Skip to content
Permalink
Browse files

Proper mime encoding of high ascii strings

Reviewed-by: ehelin
  • Loading branch information
Robin Westberg
Robin Westberg committed Sep 25, 2019
1 parent 65f5728 commit 76d0158ad22c0fd7a687a729f95d070a71d550ad
@@ -87,15 +87,16 @@ public static Email parse(String raw) {
unparsedDate = redundantTimeZonePatternMatcher.group(1);
}
var date = ZonedDateTime.parse(unparsedDate, DateTimeFormatter.RFC_1123_DATE_TIME);
var subject = message.headers.get("Subject");
var author = EmailAddress.parse(message.headers.get("From"));
var subject = MimeText.decode(message.headers.get("Subject"));
var author = EmailAddress.parse(MimeText.decode(message.headers.get("From")));
var sender = author;
if (message.headers.containsKey("Sender")) {
sender = EmailAddress.parse(message.headers.get("Sender"));
sender = EmailAddress.parse(MimeText.decode(message.headers.get("Sender")));
}
List<EmailAddress> recipients;
if (message.headers.containsKey("To")) {
recipients = Arrays.stream(message.headers.get("To").split(","))
.map(MimeText::decode)
.map(EmailAddress::parse)
.collect(Collectors.toList());
} else {
@@ -110,9 +111,10 @@ public static Email parse(String raw) {
.filter(entry -> !entry.getKey().equalsIgnoreCase("From"))
.filter(entry -> !entry.getKey().equalsIgnoreCase("Sender"))
.filter(entry -> !entry.getKey().equalsIgnoreCase("To"))
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
.collect(Collectors.toMap(Map.Entry::getKey,
entry -> MimeText.decode(entry.getValue())));

return new Email(id, date, recipients, author, sender, subject, message.body, filteredHeaders);
return new Email(id, date, recipients, author, sender, subject, MimeText.decode(message.body), filteredHeaders);
}

public static EmailBuilder create(EmailAddress author, String subject, String body) {
@@ -0,0 +1,63 @@
/*
* Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/
package org.openjdk.skara.email;

import java.io.UnsupportedEncodingException;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
import java.util.regex.Pattern;

public class MimeText {
private final static Pattern encodePattern = Pattern.compile("([^\\x00-\\x7f]+)");
private final static Pattern decodePattern = Pattern.compile("=\\?([A-Za-z0-9_.-]+)\\?([bBqQ])\\?(.*?)\\?=");
private final static Pattern decodeQuotedPrintablePattern = Pattern.compile("=([0-9A-F]{2})");

public static String encode(String raw) {
var quoteMatcher = encodePattern.matcher(raw);
return quoteMatcher.replaceAll(mo -> "=?utf-8?b?" + Base64.getEncoder().encodeToString(String.valueOf(mo.group(1)).getBytes(StandardCharsets.UTF_8)) + "?=");
}

public static String decode(String encoded) {
var quotedMatcher = decodePattern.matcher(encoded);
return quotedMatcher.replaceAll(mo -> {
try {
if (mo.group(2).toUpperCase().equals("B")) {
return new String(Base64.getDecoder().decode(mo.group(3)), mo.group(1));
} else {
var quotedPrintableMatcher = decodeQuotedPrintablePattern.matcher(mo.group(3));
return quotedPrintableMatcher.replaceAll(qmo -> {
var byteValue = new byte[1];
byteValue[0] = (byte)Integer.parseInt(qmo.group(1), 16);
try {
return new String(byteValue, mo.group(1));
} catch (UnsupportedEncodingException e) {
throw new RuntimeException(e);
}
});
}
} catch (UnsupportedEncodingException e) {
throw new RuntimeException(e);
}
});
}
}
@@ -57,17 +57,17 @@ public static void send(String server, EmailAddress recipient, Email email) thro
session.sendCommand("MAIL FROM:" + email.sender().address(), mailReply);
session.sendCommand("RCPT TO:<" + recipient.address() + ">", rcptReply);
session.sendCommand("DATA", dataReply);
session.sendCommand("From: " + email.author());
session.sendCommand("From: " + MimeText.encode(email.author().toString()));
session.sendCommand("Message-Id: " + email.id());
session.sendCommand("Date: " + email.date().format(DateTimeFormatter.RFC_1123_DATE_TIME));
session.sendCommand("Sender: " + email.sender());
session.sendCommand("To: " + recipient);
session.sendCommand("Sender: " + MimeText.encode(email.sender().toString()));
session.sendCommand("To: " + MimeText.encode(recipient.toString()));
for (var header : email.headers()) {
session.sendCommand(header + ": " + email.headerValue(header));
session.sendCommand(header + ": " + MimeText.encode(email.headerValue(header)));
}
session.sendCommand("Subject: " + email.subject());
session.sendCommand("Subject: " + MimeText.encode(email.subject()));
session.sendCommand("");
session.sendCommand(email.body());
session.sendCommand(MimeText.encode(email.body()));
session.sendCommand(".", doneReply);
session.sendCommand("QUIT");
}
@@ -112,4 +112,24 @@ void redundantTimeZone() {
assertEquals("The body", mail.body());
}

@Test
void parseEncoded() {
var mail = Email.parse("Message-Id: <a@b.c>\n" +
"Date: Wed, 27 Mar 2019 14:31:00 +0100\n" +
"Subject: hello\n" +
"From: r.b at c.d (r =?iso-8859-1?Q?b=E4?=)\n" +
"To: C <c@c.c>, <d@d.c>\n" +
"\n" +
"The body"
);

assertEquals(EmailAddress.from("a@b.c"), mail.id());
assertEquals("hello", mail.subject());
assertEquals(EmailAddress.from("r bä", "r.b@c.d"), mail.author());
assertEquals(EmailAddress.from("r bä", "r.b@c.d"), mail.sender());
assertEquals(List.of(EmailAddress.from("C", "c@c.c"),
EmailAddress.from("d@d.c")),
mail.recipients());
assertEquals("The body", mail.body());
}
}
@@ -0,0 +1,44 @@
/*
* Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/
package org.openjdk.skara.email;

import org.junit.jupiter.api.Test;

import static org.junit.jupiter.api.Assertions.assertEquals;

class MimeTextTests {
@Test
void encode() {
assertEquals("=?utf-8?b?w6XDpMO2?=", MimeText.encode("åäö"));
}

@Test
void decode() {
assertEquals("åäö", MimeText.decode("=?utf-8?b?w6XDpMO2?="));
}

@Test
void decodeIsoQ() {
assertEquals("", MimeText.decode("=?iso-8859-1?Q?B=E4?="));
}
}
@@ -67,4 +67,21 @@ void withHeader() throws IOException {
assertEquals(sentMail, email);
}
}

@Test
void encoded() throws IOException {
log.info("Hello");
try (var server = new SMTPServer()) {
var sender = EmailAddress.from("Señor Dévèlöper", "test@test.email");
var recipient = EmailAddress.from("Dêst", "dest@dest.email");
var sentMail = Email.create(sender, "Sübject", "Bödÿ")
.recipient(recipient)
.header("Something", "Öthè®")
.build();

SMTP.send(server.address(), recipient, sentMail);
var email = server.receive(Duration.ofSeconds(10));
assertEquals(sentMail, email);
}
}
}
@@ -25,7 +25,6 @@
import org.openjdk.skara.email.*;

import java.io.*;
import java.nio.charset.StandardCharsets;
import java.time.format.DateTimeFormatter;
import java.util.*;
import java.util.function.Function;
@@ -42,16 +41,13 @@
"EEE LLL dd HH:mm:ss yyyy", Locale.US);
private final static Pattern fromStringEncodePattern = Pattern.compile("^(>*From )", Pattern.MULTILINE);
private final static Pattern fromStringDecodePattern = Pattern.compile("^>(>*From )", Pattern.MULTILINE);
private final static Pattern encodeQuotedPrintablePattern = Pattern.compile("([^\\x00-\\x7f]+)");
private final static Pattern decodedQuotedPrintablePattern = Pattern.compile("=\\?utf-8\\?b\\?(.*?)\\?=");

private static List<Email> splitMbox(String mbox) {
// Initial split
var messages = mboxMessagePattern.matcher(mbox).results()
.map(match -> match.group(1))
.filter(message -> message.length() > 0)
.map(Mbox::decodeFromStrings)
.map(Mbox::decodeQuotedPrintable)
.collect(Collectors.toList());

// Pipermail can occasionally fail to encode 'From ' in message bodies, try to handle this
@@ -82,16 +78,6 @@ private static String decodeFromStrings(String body) {
return fromStringMatcher.replaceAll("$1");
}

private static String encodeQuotedPrintable(String raw) {
var quoteMatcher = encodeQuotedPrintablePattern.matcher(raw);
return quoteMatcher.replaceAll(mo -> "=?utf-8?b?" + Base64.getEncoder().encodeToString(String.valueOf(mo.group(1)).getBytes(StandardCharsets.UTF_8)) + "?=");
}

private static String decodeQuotedPrintable(String raw) {
var quotedMatcher = decodedQuotedPrintablePattern.matcher(raw);
return quotedMatcher.replaceAll(mo -> new String(Base64.getDecoder().decode(mo.group(1)), StandardCharsets.UTF_8));
}

public static List<Conversation> parseMbox(String mbox) {
var emails = splitMbox(mbox);
var idToMail = emails.stream().collect(Collectors.toMap(Email::id, Function.identity(), (a, b) -> a));
@@ -128,22 +114,23 @@ public static String fromMail(Email mail) {

mboxMail.println();
mboxMail.println("From " + mail.sender().address() + " " + mail.date().format(ctimeFormat));
mboxMail.println("From: " + mail.author().toObfuscatedString());
mboxMail.println("From: " + MimeText.encode(mail.author().toObfuscatedString()));
if (!mail.author().equals(mail.sender())) {
mboxMail.println("Sender: " + mail.sender().toObfuscatedString());
mboxMail.println("Sender: " + MimeText.encode(mail.sender().toObfuscatedString()));
}
if (!mail.recipients().isEmpty()) {
mboxMail.println("To: " + mail.recipients().stream()
.map(EmailAddress::toString)
.map(MimeText::encode)
.collect(Collectors.joining(", ")));
}
mboxMail.println("Date: " + mail.date().format(DateTimeFormatter.RFC_1123_DATE_TIME));
mboxMail.println("Subject: " + mail.subject());
mboxMail.println("Subject: " + MimeText.encode(mail.subject()));
mboxMail.println("Message-Id: " + mail.id());
mail.headers().forEach(header -> mboxMail.println(header + ": " + mail.headerValue(header)));
mail.headers().forEach(header -> mboxMail.println(header + ": " + MimeText.encode(mail.headerValue(header))));
mboxMail.println();
mboxMail.println(encodeFromStrings(mail.body()));
mboxMail.println(encodeFromStrings(MimeText.encode(mail.body())));

return encodeQuotedPrintable(mboxString.toString());
return mboxString.toString();
}
}
@@ -42,6 +42,8 @@
private static Pattern messageEndPattern = Pattern.compile("^\\.$");
private static Pattern quitPattern = Pattern.compile("^QUIT$");

private final static Pattern encodeQuotedPrintablePattern = Pattern.compile("([^\\x00-\\x7f]+)");

private class AcceptThread implements Runnable {
private void handleSession(SMTPSession session) throws IOException {
session.sendCommand("220 localhost SMTP", ehloPattern);
@@ -52,7 +54,11 @@ private void handleSession(SMTPSession session) throws IOException {
var message = session.readLinesUntil(messageEndPattern);
session.sendCommand("250 MESSAGE OK", quitPattern);

var email = Email.parse(String.join("\n", message));
// SMTP is only 7-bit safe, ensure that we break any high ascii passing through here
var quoteMatcher = encodeQuotedPrintablePattern.matcher(String.join("\n", message));
var ascii7message = quoteMatcher.replaceAll(mo -> "HIGH_ASCII");

var email = Email.parse(ascii7message);
emails.addLast(email);
}

0 comments on commit 76d0158

Please sign in to comment.
You can’t perform that action at this time.