Skip to content

Commit

Permalink
Merge pull request #31456 from code-dot-org/improve-poste-testability
Browse files Browse the repository at this point in the history 
Extract email sending logic from deliver_poste_messages cron script into lib/cdo/poste.rb
  • Loading branch information
@Hamms
Hamms committed Oct 29, 2019
2 parents a301e59 + 188e10c commit c5fb5ab
Show file tree
Hide file tree
Showing 3 changed files with 289 additions and 240 deletions.
240 changes: 0 additions & 240 deletions bin/cron/deliver_poste_messages
View file
Expand Up @@ -2,7 +2,6 @@
require File.expand_path('../../../pegasus/src/env', __FILE__)
require 'retryable'
require 'cdo/only_one'
require 'cdo/parse_email_address_string'
require 'cdo/poste'
require 'honeybadger/ruby'
require 'base64'
Expand All @@ -18,20 +17,6 @@ BATCH_SIZE = 500_000
MAX_THREAD_COUNT = 50
MIN_MESSAGES_PER_THREAD = 50

# Attempt SMTP connections up to 5 times, retrying on the following error types AND message match.
CONNECTION_ATTEMPTS = 5
RETRYABLE_ERROR_TYPES = [
Net::SMTPServerBusy,
Net::SMTPAuthenticationError,
EOFError
].freeze
RETRYABLE_ERROR_MESSAGES = [
'Too many connections, try again later',
'Temporary authentication failure',
'end of file reached'
].map(&:freeze).freeze
RETRYABLE_ERROR_MESSAGE_MATCH = Regexp.new RETRYABLE_ERROR_MESSAGES.map {|m| "(#{m})"}.join('|')

SMTP_OPTIONS = {
address: CDO.poste_smtp_server,
port: 587,
Expand All @@ -47,231 +32,6 @@ SMTP_OPTIONS = {
# domain:'code.org',
#}

MESSAGE_TEMPLATES = {}.tap do |results|
POSTE_DB[:poste_messages].all.each do |message|
results[message[:id]] = message
end
end

POSTE_BASE_URL = (rack_env?(:production) ? 'https://' : 'http://') + CDO.poste_host
def poste_url(*parts)
File.join(POSTE_BASE_URL, *parts)
end

module Poste
class Template
def initialize(body, engine=TextRender::MarkdownEngine)
if match = body.match(/^---\s*\n(?<header>.*?\n?)^(---\s*$\n?)(?<html>\s*\n.*?\n?)^(---\s*$\n?)(?<text>\s*\n.*?\n?\z)/m)
@header = TextRender::YamlEngine.new(match[:header].strip)
@html = engine.new(match[:html].strip)
@text = TextRender::ErbEngine.new(match[:text].strip)
elsif match = body.match(/^---\s*\n(?<header>.*?\n?)^(---\s*$\n?)(?<html>\s*\n.*?\n?\z)/m)
@header = TextRender::YamlEngine.new(match[:header].strip)
@html = engine.new(match[:html].strip)
else
@html = engine.new(body.strip)
end
end

def render(params={})
if params.key?('form_id')
form = Form2.from_row(DB[:forms].where(id: params['form_id']).first)
params.merge! form.data
params.merge! form.processed_data
params['form'] = form
end
locals = OpenStruct.new(params).instance_eval {binding}

header = @header.result(locals) unless @header.nil?
# TODO(andrew): Fix this so that we get a signal as to how often this is happening.
# For more information, see https://www.pivotaltracker.com/story/show/104750788.
tracking_id = header['litmus_tracking_id'] unless header.nil?

html = @html.result(locals) if @html
# Parse the html into a DOM and then re-serialize back to html text in case we were depending on that
# logic in the click tracking method to clean up or canonicalize the HTML.
html = Nokogiri::HTML(html).to_html if html
html = inject_litmus_tracking html, tracking_id, params[:encrypted_id] if html
text = @text.result(locals) unless @text.nil?

[header, html, text]
end

def inject_litmus_tracking(html, tracking_id, unique_id)
return html unless tracking_id && unique_id
litmus_blob = <<-eos
<style>@media print{ #_t { background-image: url('https://#{tracking_id}.emltrk.com/#{tracking_id}?p&d=#{unique_id}');}} div.OutlookMessageHeader {background-image:url('https://#{tracking_id}.emltrk.com/#{tracking_id}?f&d=#{unique_id}')} table.moz-email-headers-table {background-image:url('https://#{tracking_id}.emltrk.com/#{tracking_id}?f&d=#{unique_id}')} blockquote #_t {background-image:url('https://#{tracking_id}.emltrk.com/#{tracking_id}?f&d=#{unique_id}')} #MailContainerBody #_t {background-image:url('https://#{tracking_id}.emltrk.com/#{tracking_id}?f&d=#{unique_id}')}</style><div id="_t"></div>
<img src="https://#{tracking_id}.emltrk.com/#{tracking_id}?d=#{unique_id}" width="1" height="1" border="0" />
eos
html.gsub("</body>", litmus_blob + "\n</body>")
end
end
end

class Deliverer
def initialize(params)
@params = params.dup
@smtp = reset_connection
@templates = {}
end

def reset_connection
@smtp.finish if @smtp
@smtp = smtp_connect unless rack_env?(:development)
end

def send(delivery)
recipient = POSTE_DB[:contacts].where(id: delivery[:contact_id]).first
message = MESSAGE_TEMPLATES[delivery[:message_id]]
encrypted_id = Poste.encrypt_id(delivery[:id])
params = JSON.parse(delivery[:params])
unsubscribe_url = poste_url("/u/#{CGI.escape(encrypted_id)}")

header, html, _ = load_template(message[:name]).render(
params.merge(
{
recipient: OpenStruct.new(recipient),
encrypted_id: encrypted_id,
unsubscribe_link: unsubscribe_url,
tracking_pixel: poste_url("/o/#{encrypted_id}"),
}
)
)

message = StringIO.new

# Merge contact_email from the delivery for code studio students whose emails we don't store in contacts.
to_address = parse_address(header['to'], recipient.merge({temporary_email: delivery[:contact_email]}))
message.puts 'To: ' + format_address(to_address)

from_address = parse_address(header['from'], {email: 'help@code.org', name: 'Code.org'})
message.puts 'From: ' + format_address(from_address)

# List of the email part of all destination addresses, including To, Cc, and Bcc
# Note if any of these are omitted it won't be delivered to them even though they still appear in the headers.
# See https://ruby-doc.org/stdlib-2.0.0/libdoc/net/smtp/rdoc/Net/SMTP.html#method-i-send_message
# and https://stackoverflow.com/questions/2530142/ruby-netsmtp-send-email-with-bcc-recipients
to_addresses = [to_address[:email]]
['Cc', 'Bcc'].each do |field|
next unless address = parse_address(header[field.downcase])
message.puts "#{field}: #{format_address(address)}"
to_addresses << address[:email]
end

['Reply-To', 'Sender'].each do |field|
next unless address = parse_address(header[field.downcase])
message.puts "#{field}: #{format_address(address)}"
end

subject = header['subject'].to_s.strip
message.puts 'Subject: ' + subject unless subject.empty?

message.puts "X-Unsubscribe-Web: #{unsubscribe_url}"
message.puts "List-Unsubscribe: <#{unsubscribe_url}>"

message.puts 'MIME-Version: 1.0'

attachments = header['attachments'] || {}
if params['attachments']
attached_files = Poste2.load_attachments(params['attachments'])
attachments.merge! attached_files
end

marker = "==_mimepart_#{SecureRandom.hex(17)}"
message.puts "Content-Type: multipart/mixed; boundary=\"#{marker}\""

message.puts ''
message.puts "--#{marker}"

message.puts 'Content-Type: text/html; charset=UTF-8'
message.puts 'Content-Transfer-Encoding: 8bit'
message.puts ''
message.write html

unless attachments.empty?
attachments.each_pair do |filename, content|
message.puts ''
message.puts "--#{marker}"
message.puts "Content-Type: image/jpeg; charset=UTF-8; filename=\"#{filename}\""
message.puts 'Content-Transfer-Encoding: base64'
message.puts "Content-Disposition: attachment; filename=\"#{filename}\""
message.puts ''

message.write content.scan(/.{1,61}/).join("\n")
end
end

message.puts ''
message.puts "--#{marker}--"

if !rack_env?(:development)
@smtp.send_message message.string, from_address[:email], *to_addresses
else
puts(message.string)
end
end

private

def format_address(address)
email = address[:email].to_s.strip
raise ArgumentError, 'No :email' if email.empty?

name = address[:name].to_s.strip
return email if name.empty?

name = "\"#{name.tr('"', '\"').tr("'", "\'")}\"" if name =~ /[;,\"\'\(\)]/
"#{name} <#{email}>".strip
end

def load_template(name)
template = @templates[name]
return template if template

path = Poste.resolve_template(name)
raise ArgumentError, "[Poste] '#{name}' template wasn't found." unless path

engine = {
'.haml' => TextRender::HamlEngine,
'.html' => TextRender::ErbEngine,
'.md' => TextRender::MarkdownEngine,
'.txt' => TextRender::MarkdownEngine,
'.yml' => TextRender::YamlEngine,
}[File.extname(path).downcase]

@templates[name] = Poste::Template.new IO.read(path), engine
end

def parse_address(address, defaults={})
address = address.to_s.strip
return parse_email_address_string(address) unless address.empty?

# Student accounts don't have a stored email in contacts,
# so we use the temporary email here when email doesn't exist.
email = defaults[:email].to_s.strip
email = defaults[:temporary_email] if email.blank?
return nil if email.blank?

{email: email}.tap do |name_and_email|
name = defaults[:name].to_s.strip
name_and_email[:name] = name unless name.empty?
end
end

def smtp_connect
Retryable.retryable(
tries: CONNECTION_ATTEMPTS,
on: RETRYABLE_ERROR_TYPES,
matching: RETRYABLE_ERROR_MESSAGE_MATCH
) do
Net::SMTP.new(@params[:address], @params[:port]).tap do |smtp|
smtp.enable_starttls if @params[:enable_starttls_auto]
smtp.start(@params[:domain], @params[:user_name], @params[:password], @params[:authentication])
end
end
end
end

def create_threads(count)
[].tap do |threads|
count.times do
Expand Down

0 comments on commit c5fb5ab

Please sign in to comment.