Proper mime encoding of high ascii strings

Reviewed-by: ehelin

email/src/main/java/org/openjdk/skara/email/Email.java

@@ -87,15 +87,16 @@ public static Email parse(String raw) {
             unparsedDate = redundantTimeZonePatternMatcher.group(1);
         }
         var date = ZonedDateTime.parse(unparsedDate, DateTimeFormatter.RFC_1123_DATE_TIME);
-        var subject = message.headers.get("Subject");
-        var author = EmailAddress.parse(message.headers.get("From"));
+        var subject = MimeText.decode(message.headers.get("Subject"));
+        var author = EmailAddress.parse(MimeText.decode(message.headers.get("From")));
         var sender = author;
         if (message.headers.containsKey("Sender")) {
-            sender = EmailAddress.parse(message.headers.get("Sender"));
+            sender = EmailAddress.parse(MimeText.decode(message.headers.get("Sender")));
         }
         List<EmailAddress> recipients;
         if (message.headers.containsKey("To")) {
             recipients = Arrays.stream(message.headers.get("To").split(","))
+                               .map(MimeText::decode)
                                .map(EmailAddress::parse)
                                .collect(Collectors.toList());
         } else {
@@ -110,9 +111,10 @@ public static Email parse(String raw) {
                                              .filter(entry -> !entry.getKey().equalsIgnoreCase("From"))
                                              .filter(entry -> !entry.getKey().equalsIgnoreCase("Sender"))
                                              .filter(entry -> !entry.getKey().equalsIgnoreCase("To"))
-                                             .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+                                             .collect(Collectors.toMap(Map.Entry::getKey,
+                                                                       entry -> MimeText.decode(entry.getValue())));
 
-        return new Email(id, date, recipients, author, sender, subject, message.body, filteredHeaders);
+        return new Email(id, date, recipients, author, sender, subject, MimeText.decode(message.body), filteredHeaders);
     }
 
     public static EmailBuilder create(EmailAddress author, String subject, String body) {

email/src/main/java/org/openjdk/skara/email/MimeText.java

@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.openjdk.skara.email;
+
+import java.io.UnsupportedEncodingException;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+import java.util.regex.Pattern;
+
+public class MimeText {
+    private final static Pattern encodePattern = Pattern.compile("([^\\x00-\\x7f]+)");
+    private final static Pattern decodePattern = Pattern.compile("=\\?([A-Za-z0-9_.-]+)\\?([bBqQ])\\?(.*?)\\?=");
+    private final static Pattern decodeQuotedPrintablePattern = Pattern.compile("=([0-9A-F]{2})");
+
+    public static String encode(String raw) {
+        var quoteMatcher = encodePattern.matcher(raw);
+        return quoteMatcher.replaceAll(mo -> "=?utf-8?b?" + Base64.getEncoder().encodeToString(String.valueOf(mo.group(1)).getBytes(StandardCharsets.UTF_8)) + "?=");
+    }
+
+    public static String decode(String encoded) {
+        var quotedMatcher = decodePattern.matcher(encoded);
+        return quotedMatcher.replaceAll(mo -> {
+            try {
+                if (mo.group(2).toUpperCase().equals("B")) {
+                    return new String(Base64.getDecoder().decode(mo.group(3)), mo.group(1));
+                } else {
+                    var quotedPrintableMatcher = decodeQuotedPrintablePattern.matcher(mo.group(3));
+                    return quotedPrintableMatcher.replaceAll(qmo -> {
+                        var byteValue = new byte[1];
+                        byteValue[0] = (byte)Integer.parseInt(qmo.group(1), 16);
+                        try {
+                            return new String(byteValue, mo.group(1));
+                        } catch (UnsupportedEncodingException e) {
+                            throw new RuntimeException(e);
+                        }
+                    });
+                }
+            } catch (UnsupportedEncodingException e) {
+                throw new RuntimeException(e);
+            }
+        });
+    }
+}

email/src/main/java/org/openjdk/skara/email/SMTP.java

@@ -57,17 +57,17 @@ public static void send(String server, EmailAddress recipient, Email email) thro
             session.sendCommand("MAIL FROM:" + email.sender().address(), mailReply);
             session.sendCommand("RCPT TO:<" + recipient.address() + ">", rcptReply);
             session.sendCommand("DATA", dataReply);
-            session.sendCommand("From: " + email.author());
+            session.sendCommand("From: " + MimeText.encode(email.author().toString()));
             session.sendCommand("Message-Id: " + email.id());
             session.sendCommand("Date: " + email.date().format(DateTimeFormatter.RFC_1123_DATE_TIME));
-            session.sendCommand("Sender: " + email.sender());
-            session.sendCommand("To: " + recipient);
+            session.sendCommand("Sender: " + MimeText.encode(email.sender().toString()));
+            session.sendCommand("To: " + MimeText.encode(recipient.toString()));
             for (var header : email.headers()) {
-                session.sendCommand(header + ": " + email.headerValue(header));
+                session.sendCommand(header + ": " + MimeText.encode(email.headerValue(header)));
             }
-            session.sendCommand("Subject: " + email.subject());
+            session.sendCommand("Subject: " + MimeText.encode(email.subject()));
             session.sendCommand("");
-            session.sendCommand(email.body());
+            session.sendCommand(MimeText.encode(email.body()));
             session.sendCommand(".", doneReply);
             session.sendCommand("QUIT");
         }

email/src/test/java/org/openjdk/skara/email/EmailTests.java

@@ -112,4 +112,24 @@ void redundantTimeZone() {
         assertEquals("The body", mail.body());
     }
 
+    @Test
+    void parseEncoded() {
+        var mail = Email.parse("Message-Id: <a@b.c>\n" +
+                                       "Date: Wed, 27 Mar 2019 14:31:00 +0100\n" +
+                                       "Subject: hello\n" +
+                                       "From: r.b at c.d (r =?iso-8859-1?Q?b=E4?=)\n" +
+                                       "To: C <c@c.c>, <d@d.c>\n" +
+                                       "\n" +
+                                       "The body"
+        );
+
+        assertEquals(EmailAddress.from("a@b.c"), mail.id());
+        assertEquals("hello", mail.subject());
+        assertEquals(EmailAddress.from("r bä", "r.b@c.d"), mail.author());
+        assertEquals(EmailAddress.from("r bä", "r.b@c.d"), mail.sender());
+        assertEquals(List.of(EmailAddress.from("C", "c@c.c"),
+                             EmailAddress.from("d@d.c")),
+                     mail.recipients());
+        assertEquals("The body", mail.body());
+    }
 }

email/src/test/java/org/openjdk/skara/email/MimeTextTests.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.openjdk.skara.email;
+
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+class MimeTextTests {
+    @Test
+    void encode() {
+        assertEquals("=?utf-8?b?w6XDpMO2?=", MimeText.encode("åäö"));
+    }
+
+    @Test
+    void decode() {
+        assertEquals("åäö", MimeText.decode("=?utf-8?b?w6XDpMO2?="));
+    }
+
+    @Test
+    void decodeIsoQ() {
+        assertEquals("Bä", MimeText.decode("=?iso-8859-1?Q?B=E4?="));
+    }
+}

email/src/test/java/org/openjdk/skara/email/SMTPTests.java

@@ -67,4 +67,21 @@ void withHeader() throws IOException {
             assertEquals(sentMail, email);
         }
     }
+
+    @Test
+    void encoded() throws IOException {
+        log.info("Hello");
+        try (var server = new SMTPServer()) {
+            var sender = EmailAddress.from("Señor Dévèlöper", "test@test.email");
+            var recipient = EmailAddress.from("Dêst", "dest@dest.email");
+            var sentMail = Email.create(sender, "Sübject", "Bödÿ")
+                                .recipient(recipient)
+                                .header("Something", "Öthè®")
+                                .build();
+
+            SMTP.send(server.address(), recipient, sentMail);
+            var email = server.receive(Duration.ofSeconds(10));
+            assertEquals(sentMail, email);
+        }
+    }
 }

mailinglist/src/main/java/org/openjdk/skara/mailinglist/Mbox.java

@@ -25,7 +25,6 @@
 import org.openjdk.skara.email.*;
 
 import java.io.*;
-import java.nio.charset.StandardCharsets;
 import java.time.format.DateTimeFormatter;
 import java.util.*;
 import java.util.function.Function;
@@ -42,16 +41,13 @@ public class Mbox {
             "EEE LLL dd HH:mm:ss yyyy", Locale.US);
     private final static Pattern fromStringEncodePattern = Pattern.compile("^(>*From )", Pattern.MULTILINE);
     private final static Pattern fromStringDecodePattern = Pattern.compile("^>(>*From )", Pattern.MULTILINE);
-    private final static Pattern encodeQuotedPrintablePattern = Pattern.compile("([^\\x00-\\x7f]+)");
-    private final static Pattern decodedQuotedPrintablePattern = Pattern.compile("=\\?utf-8\\?b\\?(.*?)\\?=");
 
     private static List<Email> splitMbox(String mbox) {
         // Initial split
         var messages = mboxMessagePattern.matcher(mbox).results()
                                          .map(match -> match.group(1))
                                          .filter(message -> message.length() > 0)
                                          .map(Mbox::decodeFromStrings)
-                                         .map(Mbox::decodeQuotedPrintable)
                                          .collect(Collectors.toList());
 
         // Pipermail can occasionally fail to encode 'From ' in message bodies, try to handle this
@@ -82,16 +78,6 @@ private static String decodeFromStrings(String body) {
         return fromStringMatcher.replaceAll("$1");
     }
 
-    private static String encodeQuotedPrintable(String raw) {
-        var quoteMatcher = encodeQuotedPrintablePattern.matcher(raw);
-        return quoteMatcher.replaceAll(mo -> "=?utf-8?b?" + Base64.getEncoder().encodeToString(String.valueOf(mo.group(1)).getBytes(StandardCharsets.UTF_8)) + "?=");
-    }
-
-    private static String decodeQuotedPrintable(String raw) {
-        var quotedMatcher = decodedQuotedPrintablePattern.matcher(raw);
-        return quotedMatcher.replaceAll(mo -> new String(Base64.getDecoder().decode(mo.group(1)), StandardCharsets.UTF_8));
-    }
-
     public static List<Conversation> parseMbox(String mbox) {
         var emails = splitMbox(mbox);
         var idToMail = emails.stream().collect(Collectors.toMap(Email::id, Function.identity(), (a, b) -> a));
@@ -128,22 +114,23 @@ public static String fromMail(Email mail) {
 
         mboxMail.println();
         mboxMail.println("From " + mail.sender().address() + "  " + mail.date().format(ctimeFormat));
-        mboxMail.println("From: " + mail.author().toObfuscatedString());
+        mboxMail.println("From: " + MimeText.encode(mail.author().toObfuscatedString()));
         if (!mail.author().equals(mail.sender())) {
-            mboxMail.println("Sender: " + mail.sender().toObfuscatedString());
+            mboxMail.println("Sender: " + MimeText.encode(mail.sender().toObfuscatedString()));
         }
         if (!mail.recipients().isEmpty()) {
             mboxMail.println("To: " + mail.recipients().stream()
                                           .map(EmailAddress::toString)
+                                          .map(MimeText::encode)
                                           .collect(Collectors.joining(", ")));
         }
         mboxMail.println("Date: " + mail.date().format(DateTimeFormatter.RFC_1123_DATE_TIME));
-        mboxMail.println("Subject: " + mail.subject());
+        mboxMail.println("Subject: " + MimeText.encode(mail.subject()));
         mboxMail.println("Message-Id: " + mail.id());
-        mail.headers().forEach(header -> mboxMail.println(header + ": " + mail.headerValue(header)));
+        mail.headers().forEach(header -> mboxMail.println(header + ": " + MimeText.encode(mail.headerValue(header))));
         mboxMail.println();
-        mboxMail.println(encodeFromStrings(mail.body()));
+        mboxMail.println(encodeFromStrings(MimeText.encode(mail.body())));
 
-        return encodeQuotedPrintable(mboxString.toString());
+        return mboxString.toString();
     }
 }

test/src/main/java/org/openjdk/skara/test/SMTPServer.java

@@ -42,6 +42,8 @@ public class SMTPServer implements AutoCloseable {
     private static Pattern messageEndPattern = Pattern.compile("^\\.$");
     private static Pattern quitPattern = Pattern.compile("^QUIT$");
 
+    private final static Pattern encodeQuotedPrintablePattern = Pattern.compile("([^\\x00-\\x7f]+)");
+
     private class AcceptThread implements Runnable {
         private void handleSession(SMTPSession session) throws IOException {
             session.sendCommand("220 localhost SMTP", ehloPattern);
@@ -52,7 +54,11 @@ private void handleSession(SMTPSession session) throws IOException {
             var message = session.readLinesUntil(messageEndPattern);
             session.sendCommand("250 MESSAGE OK", quitPattern);
 
-            var email = Email.parse(String.join("\n", message));
+            // SMTP is only 7-bit safe, ensure that we break any high ascii passing through here
+            var quoteMatcher = encodeQuotedPrintablePattern.matcher(String.join("\n", message));
+            var ascii7message = quoteMatcher.replaceAll(mo -> "HIGH_ASCII");
+
+            var email = Email.parse(ascii7message);
             emails.addLast(email);
         }