Handle incoming mails. Closes #4016.

 * Uses Mailman gem
 * Can be configured for POP3, Maildir, or stdin (push from mailserver)
 * Maildir can be chained with fetchmail or similar to support IMAP
 * Can be run as part of the job server, or as a separate process

Change-Id: I000000000000000000000000000001
Reviewed-on: https://gerrit.instructure.com/2971
Tested-by: Hudson <hudson@instructure.com>
Reviewed-by: Brian Whitmer <brian@instructure.com>

Gemfile

@@ -20,6 +20,7 @@ gem 'jammit',         '0.6.0'
 gem 'json',           '1.5.1'
 # native xml parsing, diigo
 gem 'libxml-ruby',    '1.1.3',  :require => 'xml/libxml'
+gem 'mailman',        '0.4.0'
 gem 'mime-types',     '1.16',   :require => 'mime/types'
 # attachment_fu (even the current technoweenie one on github) does not work
 # with mini_magick 3.1

app/messages/new_discussion_entry.email.erb

@@ -10,5 +10,5 @@
 
 <%= strip_and_truncate(asset.message, :max_length => 200) %> 
 
-Join to the conversation here: 
+Join the conversation here: 
 <%= content :link %>

app/messages/new_discussion_topic.email.erb

@@ -14,5 +14,5 @@ A new discussion has been started that may be interesting to you:
 <% if asset.attachment %>File Included: <%= asset.attachment.display_name %> - <%= asset.attachment.readable_size %>
   http://<%= HostUrl.context_host(asset.context) %>/<%= asset.context.class.to_s.downcase.pluralize %>/<%= asset.context_id %>/files/<%= asset.attachment_id %>/download<% end %>
 
-Join to the conversation here: 
+Join the conversation here: 
 <%= content :link %>

app/models/context_message.rb

@@ -195,7 +195,7 @@ def reply_from(opts)
     else
       ContextMessage.create!({
         :context => self.context,
-        :user_id => user,
+        :user => user,
         :subject => subject,
         :recipients => "#{self.user_id}",
         :root_context_message_id => self.root_context_message_id || self.id,

app/models/discussion_topic.rb

@@ -238,7 +238,9 @@ def should_send_to_stream
 
   def reply_from(opts)
     user = opts[:user]
-    message = opts[:html].strip
+    message = opts[:text].strip
+    self.extend TextHelper
+    message = format_message(message).first
     user = nil unless user && self.context.users.include?(user)
     if !user
       raise "Only context participants may reply to messages"
@@ -247,8 +249,8 @@ def reply_from(opts)
     else
       DiscussionEntry.create!({
         :message => message,
-        :discussion_topic_id => self.id,
-        :user_id => user.id
+        :discussion_topic => self,
+        :user => user,
       })
     end
   end

app/models/mailman.rb → app/models/mailer.rb

@@ -16,8 +16,8 @@
 # with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 
-class Mailman < ActionMailer::Base
-  
+class Mailer < ActionMailer::Base
+
   attr_reader :email
 
   def message(m)

app/models/message.rb

@@ -285,13 +285,19 @@ def parse!(path_type=nil)
     Time.zone = old_time_zone
     self.body
   end
-
+
+  def reply_to_secure_id
+    Canvas::Security.hmac_sha1(self.id.to_s)
+  end
+
   def reply_to_address
     res = (self.forced_reply_to || nil) rescue nil
     res = nil if self.path_type == 'sms' rescue false
     res = self.from if self.context_type == 'ErrorReport'
-    res ||= HostUrl.outgoing_email_address
-    res
+    unless res
+      addr, domain = HostUrl.outgoing_email_address.split(/@/)
+      res = "#{addr}+#{self.reply_to_secure_id}-#{self.id}@#{domain}"
+    end
   end
 
   def deliver
@@ -354,7 +360,7 @@ def deliver_via_email
       logger.info "Delivering mail: #{self.inspect}"
       res = nil
       begin
-        res = Mailman.deliver_message(self)
+        res = Mailer.deliver_message(self)
       rescue Net::SMTPServerBusy => e
         @exception = e
         logger.error "Exception: #{e.class}: #{e.message}\n\t#{e.backtrace.join("\n\t")}"

config/incoming_mail.yml.example

@@ -0,0 +1,32 @@
+# See http://rubydoc.info/github/titanous/mailman/master/file/USER_GUIDE.md for
+# available options.  rails_root will be configured automatically.
+#
+# Currently, there are three ways to process incoming mail:
+#  * Fetch directly from POP3, controlled by the Canvas job server
+#    * Make sure to configure pop3, ignore stdin, and set the poll_interval to 0
+#  * Process messages by piping them to script/process_incoming_messages
+#    * Make sure to not ignore stdin
+#  * Monitor a maildir
+#    * Make sure to ignore stdin
+#    * If poll_interval is 0, it will be run periodically by the Canvas job
+#      server
+#    * If poll_interval is non-0, script/process_incoming_messages will never
+#      return, continually monitoring the directory
+
+development:
+  # defaults are to allow reading from stdin
+
+test:
+  maildir: "maildir"
+  ignore_stdin: true
+  poll_interval: 0
+
+production:
+  poll_interval: 0
+  ignore_stdin: true
+  pop3:
+    server: "pop.example.com"
+    port: 110
+    username: "user"
+    password: "password"
+    ssl: true

config/initializers/incoming_mail.rb

@@ -0,0 +1,14 @@
+# Initialize incoming email configuration. See config/incoming_mail.yml.example.
+
+config = Setting.from_config("incoming_mail") || {}
+
+Rails.configuration.to_prepare do
+  config.each do |key, value|
+    value = value.symbolize_keys if value.respond_to? :symbolize_keys
+    Mailman.config.send(key + '=', value)
+  end
+  # yes, this is lame, but setting this to real nil makes mailman assume '.',
+  # which then reloads the rails configuration (and gets an error because we
+  # try to remove a method that's already there
+  Mailman.config.rails_root = 'nil'
+end

config/periodic_jobs.rb

@@ -50,6 +50,12 @@
   StreamItem.send_later_enqueue_args(:destroy_stream_items, { :priority => Delayed::LOW_PRIORITY }, 4.weeks.ago, false)
 end
 
+if Mailman.config.poll_interval == 0 && Mailman.config.ignore_stdin == true
+  scheduler.cron '*/1 * * * *' do
+    IncomingMessageProcessor.send_later_enqueue_args(:process, { :priority => Delayed::LOW_PRIORITY })
+  end
+end
+
 if PageView.page_view_method == :cache
   # periodically pull new page views off the cache and insert them into the db
   scheduler.cron '*/5 * * * *' do

lib/incoming_message_processor.rb

@@ -0,0 +1,78 @@
+#
+# Copyright (C) 2011 Instructure, Inc.
+#
+# This file is part of Canvas.
+#
+# Canvas is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, version 3 of the License.
+#
+# Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+class IncomingMessageProcessor
+  def self.process
+    addr, domain = HostUrl.outgoing_email_address.split(/@/)
+    regex = Regexp.new("#{Regexp.escape(addr)}\\+([0-9a-f]+)-(\\d+)@#{Regexp.escape(domain)}")
+    Mailman::Application.run do
+      to regex do
+        html_body = (message.parts.find { |p| p.content_type = 'text/html' }).body.decoded if message.multipart?
+        html_body = message.body.decoded if !message.multipart? && message.content_type = 'text/html'
+        body = (message.parts.find { |p| p.content_type = 'text/plain' }).body.decoded if message.multipart?
+        body ||= message.body.decoded
+        if !html_body
+          self.extend TextHelper
+          html_body = format_message(body).first
+        end
+
+        msg = Message.find_by_id(params['captures'][1].to_i)
+        context = msg.context if msg && params['captures'][0] == msg.reply_to_secure_id
+        cc = CommunicationChannel.find_all_by_path_and_path_type(message.from, 'email')
+        user = cc.first.user if !cc.empty?
+        begin
+          if user && context && context.respond_to?(:reply_from)
+            context.reply_from({
+              :purpose => 'general',
+              :user => user,
+              :subject => message.subject,
+              :html => html_body,
+              :text => body
+            })
+          else
+            IncomingMessageProcessor.ndr(message.from.first, message.subject)
+          end
+        rescue => e
+          ErrorLogging.log_exception(:default, e, :message => "Incoming Message Failed", :params => {:from => message.from.first, :to => message.to} )
+        end
+      end
+      default do
+        IncomingMessageProcessor.ndr(message.from.first, message.subject)
+      end
+    end
+  end
+
+  def self.ndr(from, subject)
+    message = Message.create!(
+      :to => from,
+      :from => HostUrl.outgoing_email_address,
+      :subject => "Message Reply Failed: #{subject}",
+      :body => %{The message titled "#{subject}" could not be delivered.  The message was sent to an unknown mailbox address.  If you are trying to contact someone through Canvas you can try logging in to your account and sending them a message using the Inbox tool.
+
+Thank you,
+Canvas Support
+      },
+      :delay_for => 0,
+      :context => nil,
+      :path_type => 'email',
+      :from_name => "Instructure"
+    )
+    message.deliver
+  end
+
+end

lib/text_helper.rb

@@ -88,7 +88,8 @@ def format_message(message, url=nil, notification_id=nil)
     processed_lines = []
     quote_block = []
     message.split("\n").each do |line|
-      if line[0,5] == "> " || line[0,2] == "> "
+      # check for lines starting with '>'
+      if /^(>|>)/ =~ line
         quote_block << line
       else
         processed_lines << quote_clump(quote_block) if !quote_block.empty?

script/process_incoming_emails

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'config', 'environment'))
+require 'incoming_message_processor'
+
+IncomingMessageProcessor::process