JAMES-1900 Full support for header folding in ES indexing

apache · May 5, 2017 · 900fd9d · 900fd9d
1 parent f34b6fe
commit 900fd9d
Show file tree

Hide file tree

Showing 2 changed files with 60 additions and 20 deletions.
diff --git a/...ticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/HeaderCollection.java b/...ticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/HeaderCollection.java
@@ -19,12 +19,17 @@
 
 package org.apache.james.mailbox.elasticsearch.json;
 
-import com.google.common.annotations.VisibleForTesting;
+import java.time.ZonedDateTime;
-import com.google.common.base.Preconditions;
+import java.time.format.DateTimeFormatter;
-import com.google.common.collect.ArrayListMultimap;
+import java.util.HashSet;
-import com.google.common.collect.ImmutableMultimap;
+import java.util.Locale;
-import com.google.common.collect.ImmutableSet;
+import java.util.Optional;
-import com.google.common.collect.Multimap;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
 import org.apache.james.mailbox.store.search.SearchUtil;
 import org.apache.james.mime4j.codec.DecodeMonitor;
 import org.apache.james.mime4j.codec.DecoderUtil;
@@ -37,19 +42,21 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.time.ZonedDateTime;
+import com.google.common.annotations.VisibleForTesting;
-import java.time.format.DateTimeFormatter;
+import com.google.common.base.Preconditions;
-import java.util.HashSet;
+import com.google.common.collect.ArrayListMultimap;
-import java.util.Locale;
+import com.google.common.collect.ImmutableMultimap;
-import java.util.Optional;
+import com.google.common.collect.ImmutableSet;
-import java.util.Set;
+import com.google.common.collect.Multimap;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 public class HeaderCollection {
 
+    private static String sanitizeHeaderField(String headerName) {
+        return DecoderUtil.decodeEncodedWords(
+            MimeUtil.unfold(headerName),
+            DecodeMonitor.SILENT);
+    }
+
     public static class Builder {
 
         // Some sent e-mail have this form : Wed,  3 Jun 2015 09:05:46 +0000 (UTC)
@@ -83,9 +90,10 @@ private Builder() {
         public Builder add(Field field) {
             Preconditions.checkNotNull(field);
             String headerName = field.getName().toLowerCase(Locale.US);
-            String headerValue = field.getBody();
+            String sanitizedValue = sanitizeHeaderField(field.getBody());
-            headers.put(headerName, DecoderUtil.decodeEncodedWords(headerValue, DecodeMonitor.SILENT));
+
-            handleSpecificHeader(headerName, headerValue);
+            headers.put(headerName, sanitizedValue);
+            handleSpecificHeader(headerName, sanitizedValue);
             return this;
         }
 
@@ -121,7 +129,7 @@ private void handleSpecificHeader(String headerName, String headerValue) {
 
         private void manageAddressField(String headerName, String headerValue) {
             LenientAddressParser.DEFAULT
-                .parseAddressList(MimeUtil.unfold(headerValue))
+                .parseAddressList(headerValue)
                 .stream()
                 .flatMap(this::convertAddressToMailboxStream)
                 .map((mailbox) -> new EMailer(SearchUtil.getDisplayAddress(mailbox) , mailbox.getAddress()))

diff --git a/...earch/src/test/java/org/apache/james/mailbox/elasticsearch/json/HeaderCollectionTest.java b/...earch/src/test/java/org/apache/james/mailbox/elasticsearch/json/HeaderCollectionTest.java
@@ -85,6 +85,16 @@ public void displayNamesShouldBeRetrievedWhenEncodedWord() {
             .contains("Frédéric MARTIN");
     }
 
+    @Test
+    public void getHeadersShouldDecodeValues() {
+        HeaderCollection headerCollection = HeaderCollection.builder()
+            .add(new FieldImpl("From", "=?UTF-8?B?RnLDqWTDqXJpYyBNQVJUSU4=?= <fmartin@linagora.com>, Graham CROSMARIE <gcrosmarie@linagora.com>"))
+            .build();
+
+        assertThat(headerCollection.getHeaders().get("from"))
+            .containsExactly("Frédéric MARTIN <fmartin@linagora.com>, Graham CROSMARIE <gcrosmarie@linagora.com>");
+    }
+
     @Test
     public void addressWithTwoDisplayNamesOnTheSameFieldShouldBeRetrieved() {
         HeaderCollection headerCollection = HeaderCollection.builder()
@@ -94,7 +104,29 @@ public void addressWithTwoDisplayNamesOnTheSameFieldShouldBeRetrieved() {
         assertThat(headerCollection.getFromAddressSet())
             .containsOnly(new EMailer("Christophe Hamerling", "chri.hamerling@linagora.com"),
                 new EMailer("Graham CROSMARIE", "grah.crosmarie@linagora.com"));
+    }
+
+    @Test
+    public void foldedFromHeaderShouldBeSupported() {
+        HeaderCollection headerCollection = HeaderCollection.builder()
+            .add(new FieldImpl("From", "Christophe Hamerling <chri.hamerling@linagora.com>,\r\n" +
+                " Graham CROSMARIE <grah.crosmarie@linagora.com>"))
+            .build();
+
+        assertThat(headerCollection.getFromAddressSet())
+            .containsOnly(new EMailer("Christophe Hamerling", "chri.hamerling@linagora.com"),
+                new EMailer("Graham CROSMARIE", "grah.crosmarie@linagora.com"));
+    }
+
+    @Test
+    public void foldedHeaderShouldBeSupported() {
+        HeaderCollection headerCollection = HeaderCollection.builder()
+            .add(new FieldImpl("From", "Christophe Hamerling <chri.hamerling@linagora.com>,\r\n" +
+                " Graham CROSMARIE <grah.crosmarie@linagora.com>"))
+            .build();
 
+        assertThat(headerCollection.getHeaders().get("from"))
+            .containsOnly("Christophe Hamerling <chri.hamerling@linagora.com>, Graham CROSMARIE <grah.crosmarie@linagora.com>");
     }
 
     @Test