From a80c194f07281e0bd721d0609547a57996160ce0 Mon Sep 17 00:00:00 2001 From: Mike Mondragon Date: Sun, 30 Oct 2011 23:53:04 -0700 Subject: [PATCH] Big refactor, store regexp's in the YAML confs, not strings that get eval'd. Hacks for character encodings. --- conf/1nbox.net.yml | 8 ++--- conf/aliases.yml | 2 +- conf/mediamessaging.o2.co.uk.yml | 2 +- conf/messaging.nextel.com.yml | 16 ++++----- conf/mms.3ireland.ie.yml | 16 ++++----- conf/mms.ae.yml | 2 +- conf/mms.alltel.com.yml | 4 +-- conf/mms.att.net.yml | 18 +++++----- conf/mms.dobson.net.yml | 6 ++-- conf/mms.luxgsm.lu.yml | 16 ++++----- conf/mms.mobileiam.ma.yml | 12 +++---- conf/mms.mtn.co.za.yml | 8 ++--- conf/mms.myhelio.com.yml | 14 ++++---- conf/mms.netcom.no.yml | 4 +-- conf/mms.o2online.de.yml | 12 +++---- conf/mms.three.co.uk.yml | 18 +++++----- conf/mms.uscc.net.mail | 12 +++---- conf/mms.vodacom4me.co.za.yml | 8 ++--- conf/mms2r_media.yml | 40 +++++++++++------------ conf/mobile.indosat.net.id.yml | 8 ++--- conf/msg.telus.com.yml | 4 +-- conf/orangemms.net.yml | 14 ++++---- conf/pm.sprint.com.yml | 16 ++++----- conf/pxt.vodafone.net.nz.yml | 6 ++-- conf/rci.rogers.com.yml | 14 ++++---- conf/sms.sasktel.com.yml | 18 +++++----- conf/tmomail.net.yml | 44 ++++++++++++------------- conf/txt.bell.ca.yml | 4 +-- conf/unicel.com.yml | 6 ++-- conf/vmpix.com.yml | 14 ++++---- conf/vzwpix.com.yml | 12 +++---- conf/waw.plspictures.com.yml | 17 +++++----- dev_tools/anonymizer.rb | 2 +- lib/{mail_ext.rb => ext/mail.rb} | 0 lib/mms2r.rb | 5 ++- lib/mms2r/media.rb | 35 +++++++++++--------- test/test_1nbox_net.rb | 2 +- test/test_bellsouth_net.rb | 2 +- test/test_mms2r_media.rb | 56 +++++++++++++++++++------------- test/test_mms_att_net.rb | 2 +- test/test_tmomail_net.rb | 6 ++-- test/test_vzwpix_com.rb | 2 +- 42 files changed, 262 insertions(+), 245 deletions(-) rename lib/{mail_ext.rb => ext/mail.rb} (100%) diff --git a/conf/1nbox.net.yml b/conf/1nbox.net.yml index 0ceb952..83d7aaf 100644 --- a/conf/1nbox.net.yml +++ b/conf/1nbox.net.yml @@ -1,13 +1,13 @@ --- ignore: image/jpeg: - - /^.*idea_banner.jpg$/i + - !ruby/regexp /^.*idea_banner.jpg$/i text/html: - - /Message from memova<\/title>/im + - !ruby/regexp /<title>Message from memova<\/title>/im transform: number: - from - - /^([^\/@]+).*/ - - "\1" + - !ruby/regexp /^([^\/@]+).*/ + - "\\1" diff --git a/conf/aliases.yml b/conf/aliases.yml index 1d40387..e7d530f 100644 --- a/conf/aliases.yml +++ b/conf/aliases.yml @@ -1,4 +1,4 @@ ---- +--- txt.att.net: mms.att.net cingularme.com: mms.att.net mmode.com: mms.att.net diff --git a/conf/mediamessaging.o2.co.uk.yml b/conf/mediamessaging.o2.co.uk.yml index 3cb95f2..56b57e0 100644 --- a/conf/mediamessaging.o2.co.uk.yml +++ b/conf/mediamessaging.o2.co.uk.yml @@ -1,4 +1,4 @@ --- ignore: text/plain: - - /This Media Message has been sent using an O2 camera phone/mi \ No newline at end of file + - !ruby/regexp /This Media Message has been sent using an O2 camera phone/mi diff --git a/conf/messaging.nextel.com.yml b/conf/messaging.nextel.com.yml index 2879f5f..80e3c15 100644 --- a/conf/messaging.nextel.com.yml +++ b/conf/messaging.nextel.com.yml @@ -1,10 +1,10 @@ --- ignore: - text/plain: - - /^This is an MMS message. 20\d\d Sprint Nextel. All rights reserved./ - - /^This is an MMS message. Please go to http:\/\/mms.nextel.com\/mms\/anon\/do\/legacy\/viewDirect/ - - /^This is a multi-part message in MIME format./ - text/html: - - /^<html><body>\s+<center>This is an MMS message. <br><center> 20\d\d Sprint Nextel. All rights reserved<\/center><br><img/m - - /^<html><body>\s+This is an MMS message. <br><center> 20\d\d Sprint Nextel. All rights reserved/m - - /^<html><body>\s+This is an MMS message. Please go to <a href="http:\/\/mms.nextel.com\/mms\/anon\/do\/legacy\/viewDirect/m + text/plain: + - !ruby/regexp /^This is an MMS message. 20\d\d Sprint Nextel. All rights reserved./ + - !ruby/regexp /^This is an MMS message. Please go to http:\/\/mms.nextel.com\/mms\/anon\/do\/legacy\/viewDirect/ + - !ruby/regexp /^This is a multi-part message in MIME format./ + text/html: + - !ruby/regexp /^<html><body>\s+<center>This is an MMS message. <br><center> 20\d\d Sprint Nextel. All rights reserved<\/center><br><img/m + - !ruby/regexp /^<html><body>\s+This is an MMS message. <br><center> 20\d\d Sprint Nextel. All rights reserved/m + - !ruby/regexp /^<html><body>\s+This is an MMS message. Please go to <a href="http:\/\/mms.nextel.com\/mms\/anon\/do\/legacy\/viewDirect/m diff --git a/conf/mms.3ireland.ie.yml b/conf/mms.3ireland.ie.yml index be28d2e..e583add 100644 --- a/conf/mms.3ireland.ie.yml +++ b/conf/mms.3ireland.ie.yml @@ -1,17 +1,17 @@ --- ignore: image/jpeg: - - /^.*tpm.jpg$/i - - /^.*vf5.jpg$/i - - /^.*reb.jpg$/i + - !ruby/regexp /^.*tpm.jpg$/i + - !ruby/regexp /^.*vf5.jpg$/i + - !ruby/regexp /^.*reb.jpg$/i image/gif: - - /^.*vf9.gif$/i + - !ruby/regexp /^.*vf9.gif$/i text/plain: - - /^You have received an MMS message. To see the message correctly, you will need to view this message.*/im + - !ruby/regexp /^You have received an MMS message. To see the message correctly, you will need to view this message.*/im text/html: - - /Yospace Technologies Ltd - MMS-to-Email version 1-0-1-1/im + - !ruby/regexp /Yospace Technologies Ltd - MMS-to-Email version 1-0-1-1/im number: - from - - /^([^\s\/@]+).*/ - - "\1" + - !ruby/regexp /^([^\s\/@]+).*/ + - "\\1" diff --git a/conf/mms.ae.yml b/conf/mms.ae.yml index 8cb632b..b9a8d8a 100644 --- a/conf/mms.ae.yml +++ b/conf/mms.ae.yml @@ -1,5 +1,5 @@ --- ignore: application/smil: - - /^pres\.smil/i + - !ruby/regexp /^pres\.smil/i diff --git a/conf/mms.alltel.com.yml b/conf/mms.alltel.com.yml index 77707c4..ff6cdf7 100644 --- a/conf/mms.alltel.com.yml +++ b/conf/mms.alltel.com.yml @@ -4,8 +4,8 @@ ignore: - mms text/html: - mms.smil.txt - - /<html>\s+<head>\s+<title>Picture Messaging<\/title>/m - image/gif: + - !ruby/regexp /<html>\s+<head>\s+<title>Picture Messaging<\/title>/m + image/gif: - spacer.gif - header.gif - bluebar.gif diff --git a/conf/mms.att.net.yml b/conf/mms.att.net.yml index c754322..491758a 100644 --- a/conf/mms.att.net.yml +++ b/conf/mms.att.net.yml @@ -1,11 +1,11 @@ --- -transform: - text/plain: - - - /^Multimedia message$/i +transform: + text/plain: + - - !ruby/regexp /^Multimedia message$/i - "" - - - /\A(.+?)\s+--\s+=+\s+This mobile text message is brought to you by AT&T.*/im - - "\1" - - - /\A(.+?)\s+--\s+=+\s+Brought to you by, Cingular Wireless Messaging.*/im - - "\1" - - - /\A(.+?)\s+--\s+Mobile Email from a Cingular Wireless Customer.*/im - - "\1" + - - !ruby/regexp /\A(.+?)\s+--\s+=+\s+This mobile text message is brought to you by AT&T.*/im + - "\\1" + - - !ruby/regexp /\A(.+?)\s+--\s+=+\s+Brought to you by, Cingular Wireless Messaging.*/im + - "\\1" + - - !ruby/regexp /\A(.+?)\s+--\s+Mobile Email from a Cingular Wireless Customer.*/im + - "\\1" diff --git a/conf/mms.dobson.net.yml b/conf/mms.dobson.net.yml index 585cf36..fbb6b67 100644 --- a/conf/mms.dobson.net.yml +++ b/conf/mms.dobson.net.yml @@ -1,4 +1,4 @@ ---- +--- ignore: - application/smil: - - /^<smil><head><layout><root-layout width=/i + application/smil: + - !ruby/regexp /^<smil><head><layout><root-layout width=/i diff --git a/conf/mms.luxgsm.lu.yml b/conf/mms.luxgsm.lu.yml index 9dc2211..6b2a628 100644 --- a/conf/mms.luxgsm.lu.yml +++ b/conf/mms.luxgsm.lu.yml @@ -1,13 +1,13 @@ --- ignore: - image/gif: - - /^Vodafone Logo/i - text/html: - - /You have received a picture message from phone number/im - text/plain: - - /You have received a picture message from mobile phone number/im + image/gif: + - !ruby/regexp /^Vodafone Logo/i + text/html: + - !ruby/regexp /You have received a picture message from phone number/im + text/plain: + - !ruby/regexp /You have received a picture message from mobile phone number/im number: - from - - /^([^\s]+)\s.*/ - - "\1" + - !ruby/regexp /^([^\s]+)\s.*/ + - "\\1" diff --git a/conf/mms.mobileiam.ma.yml b/conf/mms.mobileiam.ma.yml index 83ec933..dbbcf8a 100644 --- a/conf/mms.mobileiam.ma.yml +++ b/conf/mms.mobileiam.ma.yml @@ -1,16 +1,16 @@ --- ignore: image/jpeg: - - /^.*html_DEFAULT_SLIDE_header.JPG$/i + - !ruby/regexp /^.*html_DEFAULT_SLIDE_header.JPG$/i image/gif: - - /^.*footer.gif$/i - - /^.*reply.gif$/i + - !ruby/regexp /^.*footer.gif$/i + - !ruby/regexp /^.*reply.gif$/i text/html: - - /.*/ + - !ruby/regexp /.*/ transform: number: - from - - /^([^\/@]+).*/ - - "\1" + - !ruby/regexp /^([^\/@]+).*/ + - "\\1" diff --git a/conf/mms.mtn.co.za.yml b/conf/mms.mtn.co.za.yml index c74eb12..c7d5654 100644 --- a/conf/mms.mtn.co.za.yml +++ b/conf/mms.mtn.co.za.yml @@ -1,13 +1,13 @@ --- ignore: image/bmp: - - /^.*logo.bmp$/i + - !ruby/regexp /^.*logo.bmp$/i text/html: - - /.*/ + - !ruby/regexp /.*/ transform: number: - from - - /^([^\/@]+).*/ - - "\1" + - !ruby/regexp /^([^\/@]+).*/ + - "\\1" diff --git a/conf/mms.myhelio.com.yml b/conf/mms.myhelio.com.yml index a550289..4bebc23 100644 --- a/conf/mms.myhelio.com.yml +++ b/conf/mms.myhelio.com.yml @@ -1,12 +1,12 @@ --- ignore: - image/gif: + image/gif: - top.gif - bottom.gif - middle_img.gif - text/html: - - /<html>\s+<head>\s+<meta http-equiv=\"Content-Type\" content=\"text\/html; charset=iso-8859-1\">\s+<title>MMS Email<\/title>/im -transform: - text/plain: - - - /Note:\s{1,2}.*?\s+message:\s{1,2}(.+)$/m - - "\1" + text/html: + - !ruby/regexp /<html>\s+<head>\s+<meta http-equiv=\"Content-Type\" content=\"text\/html; charset=iso-8859-1\">\s+<title>MMS Email<\/title>/im +transform: + text/plain: + - - !ruby/regexp /Note:\s{1,2}.*?\s+message:\s{1,2}(.+)$/m + - "\\1" diff --git a/conf/mms.netcom.no.yml b/conf/mms.netcom.no.yml index ac5666c..0159a14 100644 --- a/conf/mms.netcom.no.yml +++ b/conf/mms.netcom.no.yml @@ -1,4 +1,4 @@ --- ignore: - text/plain: - - /Meldingen er sendt fra mobiltelefon og kan kun besvares med mobiltelefon. Nummeret finner du i e-postadressen./im + text/plain: + - !ruby/regexp /Meldingen er sendt fra mobiltelefon og kan kun besvares med mobiltelefon. Nummeret finner du i e-postadressen./im diff --git a/conf/mms.o2online.de.yml b/conf/mms.o2online.de.yml index 3569478..0d1107e 100644 --- a/conf/mms.o2online.de.yml +++ b/conf/mms.o2online.de.yml @@ -1,8 +1,8 @@ --- ignore: - text/html: - - /Diese Nachricht wurde Ihnen von einem Handy mit der Nummer/im -transform: - text/plain: - - - /.*bermittelt.\s+(.+)\s+Wenn Sie mehr.*/im - - "\1" + text/html: + - !ruby/regexp /Diese Nachricht wurde Ihnen von einem Handy mit der Nummer/im +transform: + text/plain: + - - !ruby/regexp /.*bermittelt.\s+(.+)\s+Wenn Sie mehr.*/im + - "\\1" diff --git a/conf/mms.three.co.uk.yml b/conf/mms.three.co.uk.yml index 951a832..93dc5f3 100644 --- a/conf/mms.three.co.uk.yml +++ b/conf/mms.three.co.uk.yml @@ -1,15 +1,15 @@ --- -transform: - text/plain: - - - /^You have received a new message$/i +transform: + text/plain: + - - !ruby/regexp /^You have received a new message$/i - "" ignore: - image/jpeg: + image/jpeg: - images/reb.jpg - images/tpm.jpg - image/gif: + image/gif: - images/vf9.gif - text/html: - - /<html><head>.*<title>MMS Message<\/title>/im - text/plain: - - /You have received an MMS message. To see the message correctly, you will need to view this message\s+in a graphical email application./im + text/html: + - !ruby/regexp /<html><head>.*<title>MMS Message<\/title>/im + text/plain: + - !ruby/regexp /You have received an MMS message. To see the message correctly, you will need to view this message\s+in a graphical email application./im diff --git a/conf/mms.uscc.net.mail b/conf/mms.uscc.net.mail index 33b3904..1d8bb4d 100644 --- a/conf/mms.uscc.net.mail +++ b/conf/mms.uscc.net.mail @@ -1,8 +1,8 @@ --- ignore: - text/plain: - - /\AThank you for using Picture and Video Messaging by U\.S\. Cellular.*/m -transform: - text/plain: - - - /\A(.+?)\s+Thank you for using Picture and Video Messaging by U\.S\. Cellular.*/m - - "\1" + text/plain: + - !ruby/regexp /\AThank you for using Picture and Video Messaging by U\.S\. Cellular.*/m +transform: + text/plain: + - - !ruby/regexp /\A(.+?)\s+Thank you for using Picture and Video Messaging by U\.S\. Cellular.*/m + - "\\1" diff --git a/conf/mms.vodacom4me.co.za.yml b/conf/mms.vodacom4me.co.za.yml index 62c1449..e3dc791 100644 --- a/conf/mms.vodacom4me.co.za.yml +++ b/conf/mms.vodacom4me.co.za.yml @@ -1,13 +1,13 @@ --- ignore: text/html: - - /.*You have an <b>MMS<\/b> message!.*/m + - !ruby/regexp /.*You have an <b>MMS<\/b> message!.*/m transform: text/plain: - - - /\s*This is an MMS message brought to you by Vodacom\. Click here to view online presentation http:\/\/mms.vodacom4me.co.za\/.*$/ + - - !ruby/regexp /\s*This is an MMS message brought to you by Vodacom\. Click here to view online presentation http:\/\/mms.vodacom4me.co.za\/.*$/ - "" number: - from - - /^([^\s]+)\s.*/ - - "\1" + - !ruby/regexp /^([^\s]+)\s.*/ + - "\\1" diff --git a/conf/mms2r_media.yml b/conf/mms2r_media.yml index a4be793..a4341d7 100644 --- a/conf/mms2r_media.yml +++ b/conf/mms2r_media.yml @@ -1,26 +1,26 @@ --- ignore: text/plain: - - /^\(no subject\)$/i - - /\ASent via BlackBerry (from|by) .*$/im - - /\ASent from my Verizon Wireless BlackBerry$/im - - /\ASent from (my|your) iPhone.?$/im - - /\ASent on the Sprint.* Now Network.*$/im + - !ruby/regexp /^\(no subject\)$/i + - !ruby/regexp /\ASent via BlackBerry (from|by) .*$/im + - !ruby/regexp /\ASent from my Verizon Wireless BlackBerry$/im + - !ruby/regexp /\ASent from (my|your) iPhone.?$/im + - !ruby/regexp /\ASent on the Sprint.* Now Network.*$/im multipart/mixed: - - "/^Attachment: /i" -transform: - text/plain: - - - /\A(.*?)Sent via BlackBerry (from|by) .*$/im - - "\1" - - - /\A(.*?)Sent from my Verizon Wireless BlackBerry$/im - - "\1" - - - /\A(.*?)Sent from (my|your) iPhone.?$/im - - "\1" - - - /\A(.*?)\s+image\/jpeg$/im - - "\1" - - - /\A(.*?)Sent on the Sprint.* Now Network.*$/im - - "\1" -device_types: + - !ruby/regexp "/^Attachment: /i" +transform: + text/plain: + - - !ruby/regexp /\A(.*?)Sent via BlackBerry (from|by) .*$/im + - "\\1" + - - !ruby/regexp /\A(.*?)Sent from my Verizon Wireless BlackBerry$/im + - "\\1" + - - !ruby/regexp /\A(.*?)Sent from (my|your) iPhone.?$/im + - "\\1" + - - !ruby/regexp /\A(.*?)\s+image\/jpeg$/im + - "\\1" + - - !ruby/regexp /\A(.*?)Sent on the Sprint.* Now Network.*$/im + - "\\1" +device_types: headers: x-mailer: :iphone: !ruby/regexp /iPhone Mail/i @@ -29,7 +29,7 @@ device_types: :iphone: !ruby/regexp /iPhone Mail/i x-rim-org-msg-ref-id: :blackberry: !ruby/regexp /.+/ - # TODO do something about the assortment of camera models that have + # TODO do something about the assortment of camera models that have # been seen: # 1.3 Megapixel, 2.0 Megapixel, BlackBerry, CU920, G'z One TYPE-S, # Hermes, iPhone, LG8700, LSI_S5K4AAFA, Micron MT9M113 1.3MP YUV, diff --git a/conf/mobile.indosat.net.id.yml b/conf/mobile.indosat.net.id.yml index 677eaa8..693b999 100644 --- a/conf/mobile.indosat.net.id.yml +++ b/conf/mobile.indosat.net.id.yml @@ -1,14 +1,14 @@ --- ignore: text/plain: - - /\A\.\z/im + - !ruby/regexp /\A\.\z/im - . text/html: - - /I-Memova - Email Praktis. Cara Taktis. Powered by IM2./i + - !ruby/regexp /I-Memova - Email Praktis. Cara Taktis. Powered by IM2./i image/jpeg: - cp__newBanner.jpg number: - return-path - - /^([^@]+)@.*/ - - "\1" + - !ruby/regexp /^([^@]+)@.*/ + - "\\1" diff --git a/conf/msg.telus.com.yml b/conf/msg.telus.com.yml index 2052779..deddb9c 100644 --- a/conf/msg.telus.com.yml +++ b/conf/msg.telus.com.yml @@ -1,6 +1,6 @@ --- ignore: text/plain: - - /This is an MMS message. Please go to http/m + - !ruby/regexp /This is an MMS message. Please go to http/m text/html: - - /You've received a Message from a TELUS phone./m + - !ruby/regexp /You've received a Message from a TELUS phone./m diff --git a/conf/orangemms.net.yml b/conf/orangemms.net.yml index d3ebd80..5dcb4b6 100644 --- a/conf/orangemms.net.yml +++ b/conf/orangemms.net.yml @@ -1,13 +1,13 @@ --- ignore: application/smil: - - /\A<smil><head><layout><root-layout height=/m - text/plain: + - !ruby/regexp /\A<smil><head><layout><root-layout height=/m + text/plain: - banniere.txt - - /\A--\s+This Orange Multi Media Message was sent wirefree from an Orange MMS.*/mi -transform: - text/plain: - - - /^Orange UK MMS$/i + - !ruby/regexp /\A--\s+This Orange Multi Media Message was sent wirefree from an Orange MMS.*/mi +transform: + text/plain: + - - !ruby/regexp /^Orange UK MMS$/i - "" - - - /^Wiadomo\303\205\302\233\303\204\302\207 multimedialna\.$/i + - - !ruby/regexp /^Wiadomo.* multimedialna\.$/i - "" diff --git a/conf/pm.sprint.com.yml b/conf/pm.sprint.com.yml index 8cf6404..616196b 100644 --- a/conf/pm.sprint.com.yml +++ b/conf/pm.sprint.com.yml @@ -1,14 +1,14 @@ --- ignore: text/html: - - /We're sorry, this page is not available. We apologize for the inconvenience./mi - text/plain: - - /You have new [Picture|Video] Mail!\s+Click Go\/View to see now./mi -transform: - text/plain: - - - /^You have new Picture Mail!$/i + - !ruby/regexp /We're sorry, this page is not available. We apologize for the inconvenience./mi + text/plain: + - !ruby/regexp /You have new [Picture|Video] Mail!\s+Click Go\/View to see now./mi +transform: + text/plain: + - - !ruby/regexp /^You have new Picture Mail!$/i - "" - - - /^New Message$/i + - - !ruby/regexp /^New Message$/i - "" - - - /^PictureMail$/i + - - !ruby/regexp /^PictureMail$/i - "" diff --git a/conf/pxt.vodafone.net.nz.yml b/conf/pxt.vodafone.net.nz.yml index 1720065..ba17b8b 100644 --- a/conf/pxt.vodafone.net.nz.yml +++ b/conf/pxt.vodafone.net.nz.yml @@ -1,5 +1,5 @@ --- -transform: - text/plain: - - - /^You have a PXT from \d+$/i +transform: + text/plain: + - - !ruby/regexp /^You have a PXT from \d+$/i - "" diff --git a/conf/rci.rogers.com.yml b/conf/rci.rogers.com.yml index 45322f4..ab1b442 100644 --- a/conf/rci.rogers.com.yml +++ b/conf/rci.rogers.com.yml @@ -1,18 +1,18 @@ --- ignore: image/gif: - - /^.*$/i + - !ruby/regexp /^.*$/i text/plain: - - /^Uve received a video or picture msg from .*/im + - !ruby/regexp /^Uve received a video or picture msg from .*/im text/html: - - /Picture from your Rogers Wireless friend/im + - !ruby/regexp /Picture from your Rogers Wireless friend/im transform: - text/plain: - - - /^You have received a Picture Message from your Rogers Wireless.*/i + text/plain: + - - !ruby/regexp /^You have received a Picture Message from your Rogers Wireless.*/i - "" number: - from - - /^([^\/@]+).*/ - - "\1" + - !ruby/regexp /^([^\/@]+).*/ + - "\\1" diff --git a/conf/sms.sasktel.com.yml b/conf/sms.sasktel.com.yml index 9f64270..72428aa 100644 --- a/conf/sms.sasktel.com.yml +++ b/conf/sms.sasktel.com.yml @@ -1,12 +1,12 @@ --- ignore: - text/html: - - /.*/im - text/plain: - - /^You have received a New Message/i - image/gif: - - /.*/im -transform: - text/plain: - - - /^A Picture\/Video Message!$/i + text/html: + - !ruby/regexp /.*/im + text/plain: + - !ruby/regexp /^You have received a New Message/i + image/gif: + - !ruby/regexp /.*/im +transform: + text/plain: + - - !ruby/regexp /^A Picture\/Video Message!$/i - "" diff --git a/conf/tmomail.net.yml b/conf/tmomail.net.yml index 3005913..d855081 100644 --- a/conf/tmomail.net.yml +++ b/conf/tmomail.net.yml @@ -1,28 +1,28 @@ --- ignore: - image/jpeg: - - /^masthead.jpg$/i - - /^logo.jpg$/i - image/gif: - - /^dottedline350.gif$/i - - /^dottedline600.gif$/i - - /^dottedLine_350.gif$/i - - /^dottedLine_600.gif$/i - - /^spacer.gif$/i - - /^video.gif$/i - - /^audio.gif$/i - - /^tmobilelogo.gif$/i - - /^tmobilespace.gif$/i - text/html: - - /<html>\s+<head>\s+<title>T-Mobile<\/title>/m - - /<title>MMS E-Mail<\/title>/m + image/jpeg: + - !ruby/regexp /^masthead.jpg$/i + - !ruby/regexp /^logo.jpg$/i + image/gif: + - !ruby/regexp /^dottedline350.gif$/i + - !ruby/regexp /^dottedline600.gif$/i + - !ruby/regexp /^dottedLine_350.gif$/i + - !ruby/regexp /^dottedLine_600.gif$/i + - !ruby/regexp /^spacer.gif$/i + - !ruby/regexp /^video.gif$/i + - !ruby/regexp /^audio.gif$/i + - !ruby/regexp /^tmobilelogo.gif$/i + - !ruby/regexp /^tmobilespace.gif$/i + text/html: + - !ruby/regexp /<html>\s+<head>\s+<title>T-Mobile<\/title>/m + - !ruby/regexp /<title>MMS E-Mail<\/title>/m number: - from - - /^"?([^\/@]+).*/ - - "\1" + - !ruby/regexp /^"?([^\/@]+).*/ + - "\\1" -transform: - text/plain: - - - /.*\s+Text:\s+(.*?)\s+-+\s.*/im - - "\1" +transform: + text/plain: + - - !ruby/regexp /.*\s+Text:\s+(.*?)\s+-+\s.*/im + - "\\1" diff --git a/conf/txt.bell.ca.yml b/conf/txt.bell.ca.yml index baa72a2..cc57285 100644 --- a/conf/txt.bell.ca.yml +++ b/conf/txt.bell.ca.yml @@ -1,4 +1,4 @@ transform: text/html: - - [/.*(<html)/im, "\1"] - \ No newline at end of file + - - !ruby/regexp /.*(<html)/im + - "\\1" diff --git a/conf/unicel.com.yml b/conf/unicel.com.yml index c7320ac..320d712 100644 --- a/conf/unicel.com.yml +++ b/conf/unicel.com.yml @@ -1,6 +1,6 @@ --- -transform: - text/plain: - - - /^Unicel$/i +transform: + text/plain: + - - !ruby/regexp /^Unicel$/i - "" diff --git a/conf/vmpix.com.yml b/conf/vmpix.com.yml index 51ec713..a9b87d6 100644 --- a/conf/vmpix.com.yml +++ b/conf/vmpix.com.yml @@ -1,11 +1,11 @@ --- ignore: text/html: - - /This message is from\s+a Virgin Mobile user. Enjoy./mi - text/plain: - - /^This is an MMS message\.$/i + - !ruby/regexp /This message is from\s+a Virgin Mobile user. Enjoy./mi + text/plain: + - !ruby/regexp /^This is an MMS message\.$/i -transform: - text/plain: - - - /(.*?)-+\s+Sent by Text Messaging from a mobile device.*/im - - "\1" +transform: + text/plain: + - - !ruby/regexp /(.*?)-+\s+Sent by Text Messaging from a mobile device.*/im + - "\\1" diff --git a/conf/vzwpix.com.yml b/conf/vzwpix.com.yml index bdd7999..7c923f5 100644 --- a/conf/vzwpix.com.yml +++ b/conf/vzwpix.com.yml @@ -1,8 +1,8 @@ --- ignore: - text/plain: - - /\AThis message (was|has been) sent using .* from Verizon Wireless!.*/m -transform: - text/plain: - - - /\A(.+?)\s+This message (was|has been) sent using .* from Verizon Wireless!.*/m - - "\1" + text/plain: + - !ruby/regexp /\AThis message (was|has been) sent using .* from Verizon Wireless!.*/m +transform: + text/plain: + - - !ruby/regexp /\A(.+?)\s+This message (was|has been) sent using .* from Verizon Wireless!.*/m + - "\\1" diff --git a/conf/waw.plspictures.com.yml b/conf/waw.plspictures.com.yml index de1be9b..7187309 100644 --- a/conf/waw.plspictures.com.yml +++ b/conf/waw.plspictures.com.yml @@ -1,15 +1,14 @@ --- -transform: - text/plain: - - - /^You have received a Multimedia Message$/i +transform: + text/plain: + - - !ruby/regexp /^You have received a Multimedia Message$/i - "" number: - return-path - - /^([^@]+)@.*/ - - "\1" + - !ruby/regexp /^([^@]+)@.*/ + - "\\1" ignore: - text/html: - - /.*/m + text/html: + - !ruby/regexp /.*/m text/plain: - - /\AA New Photo\/Video Message.\s+Click Go\/View to see now http/im - + - !ruby/regexp /\AA New Photo\/Video Message.\s+Click Go\/View to see now http/im diff --git a/dev_tools/anonymizer.rb b/dev_tools/anonymizer.rb index aa98ea6..ba6b8b6 100755 --- a/dev_tools/anonymizer.rb +++ b/dev_tools/anonymizer.rb @@ -13,7 +13,7 @@ message.gsub!(phone_number, '2068675309') message.gsub!(email_address, 'tommy.tutone@example.com') -out = File.open(message_file, "w") +out = File.open(message_file, "wb") out.puts message out.close diff --git a/lib/mail_ext.rb b/lib/ext/mail.rb similarity index 100% rename from lib/mail_ext.rb rename to lib/ext/mail.rb diff --git a/lib/mms2r.rb b/lib/mms2r.rb index b99a31d..86be41f 100644 --- a/lib/mms2r.rb +++ b/lib/mms2r.rb @@ -52,7 +52,7 @@ def self.parse raw_mail end -%W{ mail fileutils pathname tmpdir yaml uuidtools iconv exifr }.each do |g| +%W{ yaml mail fileutils pathname tmpdir yaml uuidtools iconv exifr }.each do |g| begin require g rescue LoadError @@ -61,6 +61,9 @@ def self.parse raw_mail end end +# HAX to not deal with Psych YAML parser in Ruby >= 1.9 +#YAML::ENGINE.yamler= 'syck' if defined?(YAML::ENGINE) + require File.join(File.dirname(__FILE__), 'ext/mail') require File.join(File.dirname(__FILE__), 'ext/object') require File.join(File.dirname(__FILE__), 'mms2r/media') diff --git a/lib/mms2r/media.rb b/lib/mms2r/media.rb index 1824417..5e2d5ea 100644 --- a/lib/mms2r/media.rb +++ b/lib/mms2r/media.rb @@ -159,7 +159,7 @@ def self.domain(mail) end f = File.expand_path(File.join(self.conf_dir(), "from.yml")) - from = YAML::load_file(f) rescue {} + from = YAML::load_file(f) ret = case when from.include?(from_domain) @@ -204,11 +204,11 @@ def initialize(mail, opts={}) @default_html = nil f = File.expand_path(File.join(self.conf_dir(), "aliases.yml")) - @aliases = YAML::load_file(f) rescue {} + @aliases = YAML::load_file(f) conf = "#{@aliases[@carrier] || @carrier}.yml" f = File.expand_path(File.join(self.conf_dir(), conf)) - c = YAML::load_file(f) rescue {} + c = File.exist?(f) ? YAML::load_file(f) : {} @config = self.class.initialize_config(c) processor_module = MMS2R::CARRIERS[@carrier] @@ -228,7 +228,7 @@ def number unless @number params = config['number'] if params && params.any? && (header = mail.header[params[0]]) - @number = header.to_s.gsub(eval(params[1]), params[2]) + @number = header.to_s.gsub(params[1], params[2]) end if @number.nil? || @number.blank? @number = mail.from.first.split(/@|\//).first rescue "" @@ -383,8 +383,8 @@ def process() # :yields: media_type, file def ignore_media?(type, part) ignores = config['ignore'][type] || [] ignore = ignores.detect{ |test| filename?(part) == test} - ignore ||= ignores.detect{ |test| filename?(part) =~ eval(test) if test.index('/') == 0 } - ignore ||= ignores.detect{ |test| part.body.decoded.strip =~ eval(test) if test.index('/') == 0 } + ignore ||= ignores.detect{ |test| filename?(part) =~ test if test.is_a?(Regexp) } + ignore ||= ignores.detect{ |test| part.body.decoded.strip =~ test if test.is_a?(Regexp) } ignore ||= (part.body.decoded.strip.size == 0 ? true : nil) ignore.nil? ? false : true end @@ -408,7 +408,7 @@ def process_media(part) if part.part_type? =~ /^text\// || part.part_type? == 'application/smil' type, content = transform_text_part(part) - mode = 'w' + mode = 'wb' else if part.part_type? == 'application/octet-stream' type = type_from_filename(filename?(part)) @@ -431,22 +431,25 @@ def process_media(part) # configuration. def transform_text(type, text, original_encoding = 'ISO-8859-1') - return type, text if !config['transform'] || !(transforms = config['transform'][type]) - #convert to UTF-8 - begin - c = Iconv.new('UTF-8', original_encoding ) - utf_t = c.iconv(text) - rescue Exception => e - utf_t = text + if RUBY_VERSION < "1.9" + #convert to UTF-8 + begin + c = Iconv.new('UTF-8', original_encoding ) + utf_t = c.iconv(text) + rescue Exception => e + utf_t = text + end + else + utf_t = text.encoding == "ASCII-8BIT" ? text.force_encoding("ISO-8859-1").encode("UTF-8") : text end transforms.each do |transform| next unless transform.size == 2 p = transform.first r = transform.last - utf_t = utf_t.gsub(eval(p), r) rescue utf_t + utf_t = utf_t.gsub(p, r) rescue utf_t end return type, utf_t @@ -649,7 +652,7 @@ def self.default_ext(content_type) def self.initialize_config(c) f = File.expand_path(File.join(self.conf_dir(), "mms2r_media.yml")) - conf = YAML::load_file(f) rescue {} + conf = YAML::load_file(f) conf['ignore'] ||= {} unless conf['ignore'] conf['transform'] = {} unless conf['transform'] conf['number'] = [] unless conf['number'] diff --git a/test/test_1nbox_net.rb b/test/test_1nbox_net.rb index 3651ab6..01163a3 100644 --- a/test/test_1nbox_net.rb +++ b/test/test_1nbox_net.rb @@ -48,7 +48,7 @@ def test_image_with_extra_chars assert_not_nil mms.media['image/jpeg'] assert_equal 1, mms.media['image/jpeg'].size - assert_match(/=\?UTF-8\?Q\?Copy_of_test1\.jpg\?=/, mms.media['image/jpeg'].first) + assert_match(/\/Copy of test1\.jpg$/, mms.media['image/jpeg'].first) mms.purge end diff --git a/test/test_bellsouth_net.rb b/test/test_bellsouth_net.rb index 9cf93ac..5187fd6 100644 --- a/test/test_bellsouth_net.rb +++ b/test/test_bellsouth_net.rb @@ -11,7 +11,7 @@ def test_image_from_blackberry assert_nil mms.media['text/plain'] assert_not_nil mms.media['image/jpeg'].first - assert_match(/Windows-1252\?B\?SU1HMDAwNjUuanBn/, mms.media['image/jpeg'].first) + assert_match(/\/IMG00065\.jpg$/, mms.media['image/jpeg'].first) mms.purge end end diff --git a/test/test_mms2r_media.rb b/test/test_mms2r_media.rb index 918868a..efb9081 100644 --- a/test/test_mms2r_media.rb +++ b/test/test_mms2r_media.rb @@ -1,3 +1,5 @@ +# encoding: UTF-8 + require "test_helper" class TestMms2rMedia < Test::Unit::TestCase @@ -109,13 +111,13 @@ def test_instance_initialize_config end def test_initialize_config_contatenation - c = {'ignore' => {'text/plain' => ['/A TEST/']}, - 'transform' => {'text/plain' => ['/FOO/', '']}, - 'number' => ['from', '/^([^\s]+)\s.*/', '\1'] + c = {'ignore' => {'text/plain' => [/A TEST/]}, + 'transform' => {'text/plain' => [/FOO/, '']}, + 'number' => ['from', /^([^\s]+)\s.*/, '\1'] } config = MMS2R::Media.initialize_config(c) - assert_not_nil config['ignore']['text/plain'].detect{|v| v == '/A TEST/'} - assert_not_nil config['transform']['text/plain'].detect{|v| v == '/FOO/'} + assert_not_nil config['ignore']['text/plain'].detect{|v| v == /A TEST/} + assert_not_nil config['transform']['text/plain'].detect{|v| v == /FOO/} assert_not_nil config['number'].first == 'from' end @@ -145,7 +147,7 @@ def test_default_processor_initialize_tries_to_open_config_for_carrier YAML.expects(:load_file).at_least_once.with(mms_yaml).returns({}) YAML.expects(:load_file).at_least_once.with(aliases_yaml).returns({}) YAML.expects(:load_file).at_least_once.with(from_yaml).returns([]) - YAML.expects(:load_file).at_least_once.with(example_yaml) + YAML.expects(:load_file).never.with(example_yaml) mms = MMS2R::Media.new stub_mail end @@ -159,7 +161,7 @@ def test_mms_phone_number def test_mms_phone_number_from_config mail = stub_mail(:from => '"+2068675309" <TESTER@mms.vodacom4me.co.za>') mms = MMS2R::Media.new(mail) - mms.expects(:config).once.returns({'number' => ['from', '/^([^\s]+)\s.*/', '\1']}) + mms.expects(:config).once.returns({'number' => ['from', /^([^\s]+)\s.*/, '\1']}) assert_equal '+2068675309', mms.number end @@ -172,7 +174,7 @@ def test_mms_phone_number_with_errors end end - def test_transform_text + def test_transform_text_plain mail = stub_mail mail.stubs(:from).returns(nil) mms = MMS2R::Media.new(mail) @@ -196,19 +198,18 @@ def test_transform_text assert_equal result, mms.transform_text(type, text) # matches in config - mms.expects(:config).at_least_once.returns({'transform' => {type => [["/(hello)/", 'world']]}}) + mms.expects(:config).at_least_once.returns({'transform' => {type => [[/(hello)/, 'world']]}}) assert_equal [type, 'world'], mms.transform_text(type, text) - mms.expects(:config).at_least_once.returns({'transform' => {type => [['/^Ignore this part, (.+)/', '\1']]}}) + mms.expects(:config).at_least_once.returns({'transform' => {type => [[/^Ignore this part, (.+)/, '\1']]}}) assert_equal [type, text], mms.transform_text(type, "Ignore this part, " + text) # chaining transforms - mms.expects(:config).at_least_once.returns({'transform' => {type => [['/(hello)/', 'world'], - ['/(world)/', 'mars']]}}) + mms.expects(:config).at_least_once.returns({'transform' => {type => [[/(hello)/, 'world'], + [/(world)/, 'mars']]}}) assert_equal [type, 'mars'], mms.transform_text(type, text) # has a Iconv problem - Iconv.expects(:new).raises mms.expects(:config).at_least_once.returns({'transform' => {type => [['(hello)', 'world']]}}) assert_equal result, mms.transform_text(type, text) end @@ -223,11 +224,21 @@ def test_transform_text_to_utf8 file = mms.media['text/plain'][0] assert_not_nil file assert_equal true, File::exist?(file) - text = IO.readlines("#{file}").join - #assert_match(/D'ici un mois G\303\251orgie/, text) - assert_match(/#{Regexp.escape(fixture_data('dici_un_mois_georgie.txt').strip)}/, text) - #assert_equal("sample email message Fwd: sub D'ici un mois G\303\251orgie", mms.subject) - assert_equal("sample email message Fwd: sub #{fixture_data('dici_un_mois_georgie.txt').strip}", mms.subject) + text_lines = IO.readlines("#{file}") + text = text_lines.join + + # ASCII-8BIT -> D'ici un mois G\xC3\xA9orgie + # UTF-8 -> D'ici un mois Géorgie + + if RUBY_VERSION < "1.9" + assert_equal("D'ici un mois Géorgie body", text_lines.first.strip) + assert_equal("sample email message Fwd: sub D'ici un mois Géorgie", mms.subject) + else + #assert_equal("D'ici un mois Géorgie body", text_lines.first.strip) + assert_equal("D'ici un mois G\xE9orgie body", text_lines.first.strip) + assert_equal("sample email message Fwd: sub D'ici un mois Géorgie", mms.subject) + end + mms.purge end @@ -266,7 +277,7 @@ def test_subject_with_subject_transformed mms = MMS2R::Media.new(mail) mms.stubs(:config).returns( { 'ignore' => {}, - 'transform' => {'text/plain' => [['/Default Subject: (.+)/', '\1']]}}) + 'transform' => {'text/plain' => [[/Default Subject: (.+)/, '\1']]}}) assert_equal 'hello world', mms.subject end @@ -318,7 +329,8 @@ def test_body mms = MMS2R::Media.new stub_mail temp_big = temp_text_file("hello world") mms.stubs(:default_text).returns(File.new(temp_big)) - assert_equal "hello world", mms.body + body = mms.body + assert_equal "hello world", body end def test_body_when_html @@ -451,7 +463,7 @@ def test_filename_when_file_extension_missing_part def test_ignore_media_by_filename_regexp name = 'foo.txt' - regexp = '/foo\.txt/i' + regexp = /foo\.txt/i type = 'text/plain' mms = MMS2R::Media.new stub_mail mms.stubs(:config).returns({'ignore' => {type => [regexp, 'nil.txt']}}) @@ -470,7 +482,7 @@ def test_ignore_media_by_filename_regexp def test_ignore_media_by_regexp_on_file_content name = 'foo.txt' content = "aaaaaaahello worldbbbbbbbbb" - regexp = '/.*Hello World.*/i' + regexp = /.*Hello World.*/i type = 'text/plain' mms = MMS2R::Media.new stub_mail mms.stubs(:config).returns({'ignore' => {type => ['nil.txt', regexp]}}) diff --git a/test/test_mms_att_net.rb b/test/test_mms_att_net.rb index 7a2ac59..99b47c6 100644 --- a/test/test_mms_att_net.rb +++ b/test/test_mms_att_net.rb @@ -106,7 +106,7 @@ def test_image_from_blackberry assert_equal "Hello world", IO.readlines(mms.media['text/plain'].first).join.strip assert_not_nil mms.media['image/jpeg'].first - assert_match(/Windows-1252\?B\?QkMtV0FLRS5qcGc/, mms.media['image/jpeg'].first) + assert_match(/\/BC-WAKE\.jpg$/, mms.media['image/jpeg'].first) end def test_image_from_blackberry2 diff --git a/test/test_tmomail_net.rb b/test/test_tmomail_net.rb index 5489237..d0f79e9 100644 --- a/test/test_tmomail_net.rb +++ b/test/test_tmomail_net.rb @@ -52,7 +52,7 @@ def test_image_from_blackberry assert_nil mms.media['text/plain'] assert_not_nil mms.media['image/jpeg'].first - assert_match(/Windows-1252\?B\?SU1HMDAyMzkuanBn/, mms.media['image/jpeg'].first) + assert_match(/\/IMG00239\.jpg$/, mms.media['image/jpeg'].first) mms.purge end @@ -66,7 +66,7 @@ def test_image_from_blackberry2 assert_nil mms.media['text/plain'] assert_not_nil mms.media['image/jpeg'].first - assert_match(/Windows-1252\?B\?SU1HMDAxNDEuanBn/, mms.media['image/jpeg'].first) + assert_match(/\/IMG00141\.jpg$/, mms.media['image/jpeg'].first) mms.purge end @@ -103,7 +103,7 @@ def test_tmo_blackberry_net assert_file_size mms.media['image/jpeg'][0], 337 file = mms.default_media - assert_equal '=?Windows-1252?B?SU1HMDA0NDAuanBn?=.jpg', file.original_filename + assert_equal 'IMG00440.jpg', file.original_filename mms.purge end diff --git a/test/test_vzwpix_com.rb b/test/test_vzwpix_com.rb index 56dfae1..ba084cc 100644 --- a/test/test_vzwpix_com.rb +++ b/test/test_vzwpix_com.rb @@ -154,7 +154,7 @@ def test_image_from_blackberry assert_equal "Wonderful picture!", IO.readlines(mms.media['text/plain'].first).join.strip assert_not_nil mms.media['image/jpeg'].first - assert_match(/Windows-1252\?B\?SU1HMDAwMTYuanBn/, mms.media['image/jpeg'].first) + assert_match(/\/IMG00016\.jpg$/, mms.media['image/jpeg'].first) mms.purge end