Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Proper mime encoding of high ascii strings #161

Closed
wants to merge 1 commit into from
Closed
Changes from all commits
Commits
File filter...
Filter file types
Jump to…
Jump to file or symbol
Failed to load files and symbols.

Always

Just for now

@@ -87,15 +87,16 @@ public static Email parse(String raw) {
unparsedDate = redundantTimeZonePatternMatcher.group(1);
}
var date = ZonedDateTime.parse(unparsedDate, DateTimeFormatter.RFC_1123_DATE_TIME);
var subject = message.headers.get("Subject");
var author = EmailAddress.parse(message.headers.get("From"));
var subject = MimeText.decode(message.headers.get("Subject"));
var author = EmailAddress.parse(MimeText.decode(message.headers.get("From")));
var sender = author;
if (message.headers.containsKey("Sender")) {
sender = EmailAddress.parse(message.headers.get("Sender"));
sender = EmailAddress.parse(MimeText.decode(message.headers.get("Sender")));
}
List<EmailAddress> recipients;
if (message.headers.containsKey("To")) {
recipients = Arrays.stream(message.headers.get("To").split(","))
.map(MimeText::decode)
.map(EmailAddress::parse)
.collect(Collectors.toList());
} else {
@@ -110,9 +111,10 @@ public static Email parse(String raw) {
.filter(entry -> !entry.getKey().equalsIgnoreCase("From"))
.filter(entry -> !entry.getKey().equalsIgnoreCase("Sender"))
.filter(entry -> !entry.getKey().equalsIgnoreCase("To"))
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
.collect(Collectors.toMap(Map.Entry::getKey,
entry -> MimeText.decode(entry.getValue())));

return new Email(id, date, recipients, author, sender, subject, message.body, filteredHeaders);
return new Email(id, date, recipients, author, sender, subject, MimeText.decode(message.body), filteredHeaders);
}

public static EmailBuilder create(EmailAddress author, String subject, String body) {
@@ -0,0 +1,63 @@
/*
* Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/
package org.openjdk.skara.email;

import java.io.UnsupportedEncodingException;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
import java.util.regex.Pattern;

public class MimeText {
private final static Pattern encodePattern = Pattern.compile("([^\\x00-\\x7f]+)");
private final static Pattern decodePattern = Pattern.compile("=\\?([A-Za-z0-9_.-]+)\\?([bBqQ])\\?(.*?)\\?=");
private final static Pattern decodeQuotedPrintablePattern = Pattern.compile("=([0-9A-F]{2})");

public static String encode(String raw) {
var quoteMatcher = encodePattern.matcher(raw);
return quoteMatcher.replaceAll(mo -> "=?utf-8?b?" + Base64.getEncoder().encodeToString(String.valueOf(mo.group(1)).getBytes(StandardCharsets.UTF_8)) + "?=");
}

public static String decode(String encoded) {
var quotedMatcher = decodePattern.matcher(encoded);
return quotedMatcher.replaceAll(mo -> {
try {
if (mo.group(2).toUpperCase().equals("B")) {
return new String(Base64.getDecoder().decode(mo.group(3)), mo.group(1));
} else {
var quotedPrintableMatcher = decodeQuotedPrintablePattern.matcher(mo.group(3));
return quotedPrintableMatcher.replaceAll(qmo -> {
var byteValue = new byte[1];
byteValue[0] = (byte)Integer.parseInt(qmo.group(1), 16);
try {
return new String(byteValue, mo.group(1));
} catch (UnsupportedEncodingException e) {
throw new RuntimeException(e);
}
});
}
} catch (UnsupportedEncodingException e) {
throw new RuntimeException(e);
}
});
}
}
@@ -57,17 +57,17 @@ public static void send(String server, EmailAddress recipient, Email email) thro
session.sendCommand("MAIL FROM:" + email.sender().address(), mailReply);
session.sendCommand("RCPT TO:<" + recipient.address() + ">", rcptReply);
session.sendCommand("DATA", dataReply);
session.sendCommand("From: " + email.author());
session.sendCommand("From: " + MimeText.encode(email.author().toString()));
session.sendCommand("Message-Id: " + email.id());
session.sendCommand("Date: " + email.date().format(DateTimeFormatter.RFC_1123_DATE_TIME));
session.sendCommand("Sender: " + email.sender());
session.sendCommand("To: " + recipient);
session.sendCommand("Sender: " + MimeText.encode(email.sender().toString()));
session.sendCommand("To: " + MimeText.encode(recipient.toString()));
for (var header : email.headers()) {
session.sendCommand(header + ": " + email.headerValue(header));
session.sendCommand(header + ": " + MimeText.encode(email.headerValue(header)));
}
session.sendCommand("Subject: " + email.subject());
session.sendCommand("Subject: " + MimeText.encode(email.subject()));
session.sendCommand("");
session.sendCommand(email.body());
session.sendCommand(MimeText.encode(email.body()));
session.sendCommand(".", doneReply);
session.sendCommand("QUIT");
}
@@ -112,4 +112,24 @@ void redundantTimeZone() {
assertEquals("The body", mail.body());
}

@Test
void parseEncoded() {
var mail = Email.parse("Message-Id: <a@b.c>\n" +
"Date: Wed, 27 Mar 2019 14:31:00 +0100\n" +
"Subject: hello\n" +
"From: r.b at c.d (r =?iso-8859-1?Q?b=E4?=)\n" +
"To: C <c@c.c>, <d@d.c>\n" +
"\n" +
"The body"
);

assertEquals(EmailAddress.from("a@b.c"), mail.id());
assertEquals("hello", mail.subject());
assertEquals(EmailAddress.from("r bä", "r.b@c.d"), mail.author());
assertEquals(EmailAddress.from("r bä", "r.b@c.d"), mail.sender());
assertEquals(List.of(EmailAddress.from("C", "c@c.c"),
EmailAddress.from("d@d.c")),
mail.recipients());
assertEquals("The body", mail.body());
}
}
@@ -0,0 +1,44 @@
/*
* Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/
package org.openjdk.skara.email;

import org.junit.jupiter.api.Test;

import static org.junit.jupiter.api.Assertions.assertEquals;

class MimeTextTests {
@Test
void encode() {
assertEquals("=?utf-8?b?w6XDpMO2?=", MimeText.encode("åäö"));
}

@Test
void decode() {
assertEquals("åäö", MimeText.decode("=?utf-8?b?w6XDpMO2?="));
}

@Test
void decodeIsoQ() {
assertEquals("", MimeText.decode("=?iso-8859-1?Q?B=E4?="));
}
}
@@ -67,4 +67,21 @@ void withHeader() throws IOException {
assertEquals(sentMail, email);
}
}

@Test
void encoded() throws IOException {
log.info("Hello");
try (var server = new SMTPServer()) {
var sender = EmailAddress.from("Señor Dévèlöper", "test@test.email");
var recipient = EmailAddress.from("Dêst", "dest@dest.email");
var sentMail = Email.create(sender, "Sübject", "Bödÿ")
.recipient(recipient)
.header("Something", "Öthè®")
.build();

SMTP.send(server.address(), recipient, sentMail);
var email = server.receive(Duration.ofSeconds(10));
assertEquals(sentMail, email);
}
}
}
@@ -25,7 +25,6 @@
import org.openjdk.skara.email.*;

import java.io.*;
import java.nio.charset.StandardCharsets;
import java.time.format.DateTimeFormatter;
import java.util.*;
import java.util.function.Function;
@@ -42,16 +41,13 @@
"EEE LLL dd HH:mm:ss yyyy", Locale.US);
private final static Pattern fromStringEncodePattern = Pattern.compile("^(>*From )", Pattern.MULTILINE);
private final static Pattern fromStringDecodePattern = Pattern.compile("^>(>*From )", Pattern.MULTILINE);
private final static Pattern encodeQuotedPrintablePattern = Pattern.compile("([^\\x00-\\x7f]+)");
private final static Pattern decodedQuotedPrintablePattern = Pattern.compile("=\\?utf-8\\?b\\?(.*?)\\?=");

private static List<Email> splitMbox(String mbox) {
// Initial split
var messages = mboxMessagePattern.matcher(mbox).results()
.map(match -> match.group(1))
.filter(message -> message.length() > 0)
.map(Mbox::decodeFromStrings)
.map(Mbox::decodeQuotedPrintable)
.collect(Collectors.toList());

// Pipermail can occasionally fail to encode 'From ' in message bodies, try to handle this
@@ -82,16 +78,6 @@ private static String decodeFromStrings(String body) {
return fromStringMatcher.replaceAll("$1");
}

private static String encodeQuotedPrintable(String raw) {
var quoteMatcher = encodeQuotedPrintablePattern.matcher(raw);
return quoteMatcher.replaceAll(mo -> "=?utf-8?b?" + Base64.getEncoder().encodeToString(String.valueOf(mo.group(1)).getBytes(StandardCharsets.UTF_8)) + "?=");
}

private static String decodeQuotedPrintable(String raw) {
var quotedMatcher = decodedQuotedPrintablePattern.matcher(raw);
return quotedMatcher.replaceAll(mo -> new String(Base64.getDecoder().decode(mo.group(1)), StandardCharsets.UTF_8));
}

public static List<Conversation> parseMbox(String mbox) {
var emails = splitMbox(mbox);
var idToMail = emails.stream().collect(Collectors.toMap(Email::id, Function.identity(), (a, b) -> a));
@@ -128,22 +114,23 @@ public static String fromMail(Email mail) {

mboxMail.println();
mboxMail.println("From " + mail.sender().address() + " " + mail.date().format(ctimeFormat));
mboxMail.println("From: " + mail.author().toObfuscatedString());
mboxMail.println("From: " + MimeText.encode(mail.author().toObfuscatedString()));
if (!mail.author().equals(mail.sender())) {
mboxMail.println("Sender: " + mail.sender().toObfuscatedString());
mboxMail.println("Sender: " + MimeText.encode(mail.sender().toObfuscatedString()));
}
if (!mail.recipients().isEmpty()) {
mboxMail.println("To: " + mail.recipients().stream()
.map(EmailAddress::toString)
.map(MimeText::encode)
.collect(Collectors.joining(", ")));
}
mboxMail.println("Date: " + mail.date().format(DateTimeFormatter.RFC_1123_DATE_TIME));
mboxMail.println("Subject: " + mail.subject());
mboxMail.println("Subject: " + MimeText.encode(mail.subject()));
mboxMail.println("Message-Id: " + mail.id());
mail.headers().forEach(header -> mboxMail.println(header + ": " + mail.headerValue(header)));
mail.headers().forEach(header -> mboxMail.println(header + ": " + MimeText.encode(mail.headerValue(header))));
mboxMail.println();
mboxMail.println(encodeFromStrings(mail.body()));
mboxMail.println(encodeFromStrings(MimeText.encode(mail.body())));

return encodeQuotedPrintable(mboxString.toString());
return mboxString.toString();
}
}
@@ -42,6 +42,8 @@
private static Pattern messageEndPattern = Pattern.compile("^\\.$");
private static Pattern quitPattern = Pattern.compile("^QUIT$");

private final static Pattern encodeQuotedPrintablePattern = Pattern.compile("([^\\x00-\\x7f]+)");

private class AcceptThread implements Runnable {
private void handleSession(SMTPSession session) throws IOException {
session.sendCommand("220 localhost SMTP", ehloPattern);
@@ -52,7 +54,11 @@ private void handleSession(SMTPSession session) throws IOException {
var message = session.readLinesUntil(messageEndPattern);
session.sendCommand("250 MESSAGE OK", quitPattern);

var email = Email.parse(String.join("\n", message));
// SMTP is only 7-bit safe, ensure that we break any high ascii passing through here
var quoteMatcher = encodeQuotedPrintablePattern.matcher(String.join("\n", message));
var ascii7message = quoteMatcher.replaceAll(mo -> "HIGH_ASCII");

var email = Email.parse(ascii7message);
emails.addLast(email);
}

ProTip! Use n and p to navigate between commits in a pull request.
You can’t perform that action at this time.