Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Extract email sending logic from deliver_poste_messages cron script into lib/cdo/poste.rb #31456

Merged
merged 7 commits into from Oct 29, 2019
Merged

Extract email sending logic from deliver_poste_messages cron script into lib/cdo/poste.rb #31456

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
a36600e
Extract email sending logic from deliver_poste_messages cron script i…
Hamms Oct 24, 2019
93765a3
add some missed imports
Hamms Oct 25, 2019
6b3df36
add tests
Hamms Oct 25, 2019
3966745
lazily-populate the MESSAGE_TEMPLATES constant
Hamms Oct 25, 2019
8d07624
add message data in our test rather than relying on it to already exist
Hamms Oct 25, 2019
ccd18c0
put Deliverer test in the appropriate directory
Hamms Oct 29, 2019
188e10c
add attr_readers for the send_message variables so we don't have to d…
Hamms Oct 29, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
240 changes: 0 additions & 240 deletions bin/cron/deliver_poste_messages
View file
Open in desktop
Expand Up @@ -2,7 +2,6 @@
require File.expand_path('../../../pegasus/src/env', __FILE__)
require 'retryable'
require 'cdo/only_one'
require 'cdo/parse_email_address_string'
require 'cdo/poste'
require 'honeybadger/ruby'
require 'base64'
Expand All @@ -18,20 +17,6 @@ BATCH_SIZE = 500_000
MAX_THREAD_COUNT = 50
MIN_MESSAGES_PER_THREAD = 50

# Attempt SMTP connections up to 5 times, retrying on the following error types AND message match.
CONNECTION_ATTEMPTS = 5
RETRYABLE_ERROR_TYPES = [
Net::SMTPServerBusy,
Net::SMTPAuthenticationError,
EOFError
].freeze
RETRYABLE_ERROR_MESSAGES = [
'Too many connections, try again later',
'Temporary authentication failure',
'end of file reached'
].map(&:freeze).freeze
RETRYABLE_ERROR_MESSAGE_MATCH = Regexp.new RETRYABLE_ERROR_MESSAGES.map {|m| "(#{m})"}.join('|')

SMTP_OPTIONS = {
address: CDO.poste_smtp_server,
port: 587,
Expand All @@ -47,231 +32,6 @@ SMTP_OPTIONS = {
# domain:'code.org',
#}

MESSAGE_TEMPLATES = {}.tap do |results|
POSTE_DB[:poste_messages].all.each do |message|
results[message[:id]] = message
end
end

POSTE_BASE_URL = (rack_env?(:production) ? 'https://' : 'http://') + CDO.poste_host
def poste_url(*parts)
File.join(POSTE_BASE_URL, *parts)
end

module Poste
class Template
def initialize(body, engine=TextRender::MarkdownEngine)
if match = body.match(/^---\s*\n(?<header>.*?\n?)^(---\s*$\n?)(?<html>\s*\n.*?\n?)^(---\s*$\n?)(?<text>\s*\n.*?\n?\z)/m)
@header = TextRender::YamlEngine.new(match[:header].strip)
@html = engine.new(match[:html].strip)
@text = TextRender::ErbEngine.new(match[:text].strip)
elsif match = body.match(/^---\s*\n(?<header>.*?\n?)^(---\s*$\n?)(?<html>\s*\n.*?\n?\z)/m)
@header = TextRender::YamlEngine.new(match[:header].strip)
@html = engine.new(match[:html].strip)
else
@html = engine.new(body.strip)
end
end

def render(params={})
if params.key?('form_id')
form = Form2.from_row(DB[:forms].where(id: params['form_id']).first)
params.merge! form.data
params.merge! form.processed_data
params['form'] = form
end
locals = OpenStruct.new(params).instance_eval {binding}

header = @header.result(locals) unless @header.nil?
# TODO(andrew): Fix this so that we get a signal as to how often this is happening.
# For more information, see https://www.pivotaltracker.com/story/show/104750788.
tracking_id = header['litmus_tracking_id'] unless header.nil?

html = @html.result(locals) if @html
# Parse the html into a DOM and then re-serialize back to html text in case we were depending on that
# logic in the click tracking method to clean up or canonicalize the HTML.
html = Nokogiri::HTML(html).to_html if html
html = inject_litmus_tracking html, tracking_id, params[:encrypted_id] if html
text = @text.result(locals) unless @text.nil?

[header, html, text]
end

def inject_litmus_tracking(html, tracking_id, unique_id)
return html unless tracking_id && unique_id
litmus_blob = <<-eos
<style>@media print{ #_t { background-image: url('https://#{tracking_id}.emltrk.com/#{tracking_id}?p&d=#{unique_id}');}} div.OutlookMessageHeader {background-image:url('https://#{tracking_id}.emltrk.com/#{tracking_id}?f&d=#{unique_id}')} table.moz-email-headers-table {background-image:url('https://#{tracking_id}.emltrk.com/#{tracking_id}?f&d=#{unique_id}')} blockquote #_t {background-image:url('https://#{tracking_id}.emltrk.com/#{tracking_id}?f&d=#{unique_id}')} #MailContainerBody #_t {background-image:url('https://#{tracking_id}.emltrk.com/#{tracking_id}?f&d=#{unique_id}')}</style><div id="_t"></div>
<img src="https://#{tracking_id}.emltrk.com/#{tracking_id}?d=#{unique_id}" width="1" height="1" border="0" />
eos
html.gsub("</body>", litmus_blob + "\n</body>")
end
end
end

class Deliverer
def initialize(params)
@params = params.dup
@smtp = reset_connection
@templates = {}
end

def reset_connection
@smtp.finish if @smtp
@smtp = smtp_connect unless rack_env?(:development)
end

def send(delivery)
recipient = POSTE_DB[:contacts].where(id: delivery[:contact_id]).first
message = MESSAGE_TEMPLATES[delivery[:message_id]]
encrypted_id = Poste.encrypt_id(delivery[:id])
params = JSON.parse(delivery[:params])
unsubscribe_url = poste_url("/u/#{CGI.escape(encrypted_id)}")

header, html, _ = load_template(message[:name]).render(
params.merge(
{
recipient: OpenStruct.new(recipient),
encrypted_id: encrypted_id,
unsubscribe_link: unsubscribe_url,
tracking_pixel: poste_url("/o/#{encrypted_id}"),
}
)
)

message = StringIO.new

# Merge contact_email from the delivery for code studio students whose emails we don't store in contacts.
to_address = parse_address(header['to'], recipient.merge({temporary_email: delivery[:contact_email]}))
message.puts 'To: ' + format_address(to_address)

from_address = parse_address(header['from'], {email: 'help@code.org', name: 'Code.org'})
message.puts 'From: ' + format_address(from_address)

# List of the email part of all destination addresses, including To, Cc, and Bcc
# Note if any of these are omitted it won't be delivered to them even though they still appear in the headers.
# See https://ruby-doc.org/stdlib-2.0.0/libdoc/net/smtp/rdoc/Net/SMTP.html#method-i-send_message
# and https://stackoverflow.com/questions/2530142/ruby-netsmtp-send-email-with-bcc-recipients
to_addresses = [to_address[:email]]
['Cc', 'Bcc'].each do |field|
next unless address = parse_address(header[field.downcase])
message.puts "#{field}: #{format_address(address)}"
to_addresses << address[:email]
end

['Reply-To', 'Sender'].each do |field|
next unless address = parse_address(header[field.downcase])
message.puts "#{field}: #{format_address(address)}"
end

subject = header['subject'].to_s.strip
message.puts 'Subject: ' + subject unless subject.empty?

message.puts "X-Unsubscribe-Web: #{unsubscribe_url}"
message.puts "List-Unsubscribe: <#{unsubscribe_url}>"

message.puts 'MIME-Version: 1.0'

attachments = header['attachments'] || {}
if params['attachments']
attached_files = Poste2.load_attachments(params['attachments'])
attachments.merge! attached_files
end

marker = "==_mimepart_#{SecureRandom.hex(17)}"
message.puts "Content-Type: multipart/mixed; boundary=\"#{marker}\""

message.puts ''
message.puts "--#{marker}"

message.puts 'Content-Type: text/html; charset=UTF-8'
message.puts 'Content-Transfer-Encoding: 8bit'
message.puts ''
message.write html

unless attachments.empty?
attachments.each_pair do |filename, content|
message.puts ''
message.puts "--#{marker}"
message.puts "Content-Type: image/jpeg; charset=UTF-8; filename=\"#{filename}\""
message.puts 'Content-Transfer-Encoding: base64'
message.puts "Content-Disposition: attachment; filename=\"#{filename}\""
message.puts ''

message.write content.scan(/.{1,61}/).join("\n")
end
end

message.puts ''
message.puts "--#{marker}--"

if !rack_env?(:development)
@smtp.send_message message.string, from_address[:email], *to_addresses
else
puts(message.string)
end
end

private

def format_address(address)
email = address[:email].to_s.strip
raise ArgumentError, 'No :email' if email.empty?

name = address[:name].to_s.strip
return email if name.empty?

name = "\"#{name.tr('"', '\"').tr("'", "\'")}\"" if name =~ /[;,\"\'\(\)]/
"#{name} <#{email}>".strip
end

def load_template(name)
template = @templates[name]
return template if template

path = Poste.resolve_template(name)
raise ArgumentError, "[Poste] '#{name}' template wasn't found." unless path

engine = {
'.haml' => TextRender::HamlEngine,
'.html' => TextRender::ErbEngine,
'.md' => TextRender::MarkdownEngine,
'.txt' => TextRender::MarkdownEngine,
'.yml' => TextRender::YamlEngine,
}[File.extname(path).downcase]

@templates[name] = Poste::Template.new IO.read(path), engine
end

def parse_address(address, defaults={})
address = address.to_s.strip
return parse_email_address_string(address) unless address.empty?

# Student accounts don't have a stored email in contacts,
# so we use the temporary email here when email doesn't exist.
email = defaults[:email].to_s.strip
email = defaults[:temporary_email] if email.blank?
return nil if email.blank?

{email: email}.tap do |name_and_email|
name = defaults[:name].to_s.strip
name_and_email[:name] = name unless name.empty?
end
end

def smtp_connect
Retryable.retryable(
tries: CONNECTION_ATTEMPTS,
on: RETRYABLE_ERROR_TYPES,
matching: RETRYABLE_ERROR_MESSAGE_MATCH
) do
Net::SMTP.new(@params[:address], @params[:port]).tap do |smtp|
smtp.enable_starttls if @params[:enable_starttls_auto]
smtp.start(@params[:domain], @params[:user_name], @params[:password], @params[:authentication])
end
end
end
end

def create_threads(count)
[].tap do |threads|
count.times do
Expand Down