From 7a893c49ad49a978689afd94a1e5b0c5f612cd5f Mon Sep 17 00:00:00 2001 From: Benoit Tellier Date: Thu, 22 Dec 2016 11:53:22 +0700 Subject: [PATCH] JAMES-1894 ElasticSearch should index MessageId --- .../elasticsearch/NodeMappingFactory.java | 6 +++++- .../elasticsearch/json/IndexableMessage.java | 18 +++++++++++++----- .../json/JsonMessageConstants.java | 3 ++- .../query/CriterionConverter.java | 2 +- .../elasticsearch/query/SortConverter.java | 4 ++-- .../search/ElasticSearchSearcher.java | 4 ++-- .../MailboxMessageToElasticSearchJsonTest.java | 6 +++--- .../store/src/test/resources/eml/htmlMail.json | 3 ++- mailbox/store/src/test/resources/eml/mail.json | 3 ++- .../src/test/resources/eml/nonTextual.json | 3 ++- .../src/test/resources/eml/pgpSignedMail.json | 3 ++- .../src/test/resources/eml/recursiveMail.json | 3 ++- .../eml/recursiveMailWithoutAttachments.json | 3 ++- .../store/src/test/resources/eml/spamMail.json | 3 ++- 14 files changed, 42 insertions(+), 22 deletions(-) diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/NodeMappingFactory.java b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/NodeMappingFactory.java index 7ee7edae241..751452cf39d 100644 --- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/NodeMappingFactory.java +++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/NodeMappingFactory.java @@ -70,7 +70,11 @@ private static XContentBuilder getMappingContent() { .startObject(ElasticSearchIndexer.MESSAGE_TYPE) .startObject(PROPERTIES) - .startObject(JsonMessageConstants.ID) + .startObject(JsonMessageConstants.MESSAGE_ID) + .field(TYPE, STRING) + .field(INDEX, NOT_ANALYZED) + .endObject() + .startObject(JsonMessageConstants.UID) .field(TYPE, LONG) .endObject() .startObject(JsonMessageConstants.MODSEQ) diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessage.java b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessage.java index 0896140c958..dec8b8ef885 100644 --- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessage.java +++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessage.java @@ -33,6 +33,7 @@ import org.apache.james.mailbox.elasticsearch.IndexAttachments; import org.apache.james.mailbox.elasticsearch.query.DateResolutionFormater; import org.apache.james.mailbox.extractor.TextExtractor; +import org.apache.james.mailbox.model.MessageId; import org.apache.james.mailbox.store.mail.model.MailboxMessage; import org.apache.james.mailbox.store.mail.model.Property; import org.apache.james.mime4j.MimeException; @@ -89,7 +90,8 @@ private void copyHeaderFields(HeaderCollection headerCollection, ZonedDateTime i } private void copyMessageFields(MailboxMessage message, ZoneId zoneId) { - this.id = message.getUid(); + this.messageId = message.getMessageId(); + this.uid = message.getUid(); this.mailboxId = message.getMailboxId().serialize(); this.modSeq = message.getModSeq(); this.size = message.getFullContentOctets(); @@ -127,7 +129,8 @@ private void generateText() { .collect(Collectors.joining(" ")); } - private MessageUid id; + private MessageId messageId; + private MessageUid uid; private String mailboxId; private List users; private long modSeq; @@ -156,9 +159,14 @@ private void generateText() { private Optional bodyHtml; private String text; - @JsonProperty(JsonMessageConstants.ID) - public Long getId() { - return id.asLong(); + @JsonProperty(JsonMessageConstants.MESSAGE_ID) + public String getId() { + return messageId.serialize(); + } + + @JsonProperty(JsonMessageConstants.UID) + public Long getUid() { + return uid.asLong(); } @JsonProperty(JsonMessageConstants.MAILBOX_ID) diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/JsonMessageConstants.java b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/JsonMessageConstants.java index de91b894188..f93105110da 100644 --- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/JsonMessageConstants.java +++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/JsonMessageConstants.java @@ -24,7 +24,8 @@ public interface JsonMessageConstants { /* Properties defined by JMAP */ - String ID = "id"; + String MESSAGE_ID = "messageId"; + String UID = "uid"; String MAILBOX_ID = "mailboxId"; String USERS = "users"; String IS_UNREAD = "isUnread"; diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/query/CriterionConverter.java b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/query/CriterionConverter.java index 180bf99c227..3904a92515c 100644 --- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/query/CriterionConverter.java +++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/query/CriterionConverter.java @@ -229,7 +229,7 @@ private BoolQueryBuilder convertUid(SearchQuery.UidCriterion uidCriterion) { } private QueryBuilder uidRangeFilter(SearchQuery.UidRange numericRange) { - return rangeQuery(JsonMessageConstants.ID) + return rangeQuery(JsonMessageConstants.UID) .lte(numericRange.getHighValue().asLong()) .gte(numericRange.getLowValue().asLong()); } diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/query/SortConverter.java b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/query/SortConverter.java index 2c80ecb5b27..05bc3a87569 100644 --- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/query/SortConverter.java +++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/query/SortConverter.java @@ -57,7 +57,7 @@ private static FieldSortBuilder getSortClause(SearchQuery.Sort.SortClause clause case SentDate : return SortBuilders.fieldSort(JsonMessageConstants.SENT_DATE); case Uid : - return SortBuilders.fieldSort(JsonMessageConstants.ID); + return SortBuilders.fieldSort(JsonMessageConstants.UID); case DisplayFrom: return SortBuilders.fieldSort(JsonMessageConstants.FROM + PATH_SEPARATOR + JsonMessageConstants.EMailer.NAME + PATH_SEPARATOR + NodeMappingFactory.RAW) .setNestedPath(JsonMessageConstants.FROM); @@ -65,7 +65,7 @@ private static FieldSortBuilder getSortClause(SearchQuery.Sort.SortClause clause return SortBuilders.fieldSort(JsonMessageConstants.TO + PATH_SEPARATOR + JsonMessageConstants.EMailer.NAME + PATH_SEPARATOR + NodeMappingFactory.RAW) .setNestedPath(JsonMessageConstants.TO); case Id: - return SortBuilders.fieldSort(JsonMessageConstants.ID); + return SortBuilders.fieldSort(JsonMessageConstants.MESSAGE_ID); default: throw new RuntimeException("Sort is not implemented"); } diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/ElasticSearchSearcher.java b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/ElasticSearchSearcher.java index 3b2c01244f7..1a5cd1e5bb1 100644 --- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/ElasticSearchSearcher.java +++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/ElasticSearchSearcher.java @@ -85,7 +85,7 @@ private SearchRequestBuilder getSearchRequestBuilder(Client client, List u client.prepareSearch(ElasticSearchIndexer.MAILBOX_INDEX) .setTypes(ElasticSearchIndexer.MESSAGE_TYPE) .setScroll(TIMEOUT) - .addFields(JsonMessageConstants.ID, JsonMessageConstants.MAILBOX_ID) + .addFields(JsonMessageConstants.UID, JsonMessageConstants.MAILBOX_ID, JsonMessageConstants.MESSAGE_ID) .setQuery(queryConverter.from(users, query)) .setSize(size), (searchBuilder, sort) -> searchBuilder.addSort(SortConverter.convertSort(sort)), @@ -101,7 +101,7 @@ private Stream> transformResponseToUidStream(SearchR private Optional> extractContentFromHit(SearchHit hit) { SearchHitField mailboxId = hit.field(JsonMessageConstants.MAILBOX_ID); - SearchHitField uid = hit.field(JsonMessageConstants.ID); + SearchHitField uid = hit.field(JsonMessageConstants.UID); if (mailboxId != null && uid != null) { Number uidAsNumber = uid.getValue(); return Optional.of(Pair.of(mailboxIdFactory.fromString(mailboxId.getValue()), MessageUid.of(uidAsNumber.longValue()))); diff --git a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/MailboxMessageToElasticSearchJsonTest.java b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/MailboxMessageToElasticSearchJsonTest.java index 4ee2e78503c..596d77405f6 100644 --- a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/MailboxMessageToElasticSearchJsonTest.java +++ b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/MailboxMessageToElasticSearchJsonTest.java @@ -35,13 +35,13 @@ import org.apache.commons.io.IOUtils; import org.apache.james.mailbox.FlagsBuilder; import org.apache.james.mailbox.MailboxSession.User; -import org.apache.james.mailbox.elasticsearch.IndexAttachments; import org.apache.james.mailbox.MessageUid; +import org.apache.james.mailbox.elasticsearch.IndexAttachments; import org.apache.james.mailbox.mock.MockMailboxSession; import org.apache.james.mailbox.model.MessageId; import org.apache.james.mailbox.model.TestId; +import org.apache.james.mailbox.model.TestMessageId; import org.apache.james.mailbox.store.extractor.DefaultTextExtractor; -import org.apache.james.mailbox.store.mail.model.DefaultMessageId; import org.apache.james.mailbox.store.mail.model.MailboxMessage; import org.apache.james.mailbox.store.mail.model.impl.PropertyBuilder; import org.apache.james.mailbox.store.mail.model.impl.SimpleMailboxMessage; @@ -58,7 +58,7 @@ public class MailboxMessageToElasticSearchJsonTest { public static final int SIZE = 25; public static final int BODY_START_OCTET = 100; public static final TestId MAILBOX_ID = TestId.of(18L); - public static final MessageId MESSAGE_ID = new DefaultMessageId(); + public static final MessageId MESSAGE_ID = TestMessageId.of(184L); public static final long MOD_SEQ = 42L; public static final MessageUid UID = MessageUid.of(25); public static final Charset CHARSET = Charsets.UTF_8; diff --git a/mailbox/store/src/test/resources/eml/htmlMail.json b/mailbox/store/src/test/resources/eml/htmlMail.json index 942ba1f2f64..fdc1465c4c9 100644 --- a/mailbox/store/src/test/resources/eml/htmlMail.json +++ b/mailbox/store/src/test/resources/eml/htmlMail.json @@ -1,5 +1,6 @@ { - "id":25, + "id":"184", + "uid":25, "mailboxId":"18", "modSeq":42, "size":25, diff --git a/mailbox/store/src/test/resources/eml/mail.json b/mailbox/store/src/test/resources/eml/mail.json index ed0f5c0b655..c734e45016f 100644 --- a/mailbox/store/src/test/resources/eml/mail.json +++ b/mailbox/store/src/test/resources/eml/mail.json @@ -1,5 +1,6 @@ { - "id": 25, + "id":"184", + "uid":25, "mailboxId": "18", "modSeq": 42, "size": 25, diff --git a/mailbox/store/src/test/resources/eml/nonTextual.json b/mailbox/store/src/test/resources/eml/nonTextual.json index 0cdffc08793..f15f412a726 100644 --- a/mailbox/store/src/test/resources/eml/nonTextual.json +++ b/mailbox/store/src/test/resources/eml/nonTextual.json @@ -1,5 +1,6 @@ { - "id": 25, + "id":"184", + "uid":25, "mailboxId":"18", "modSeq":42, "size":25, diff --git a/mailbox/store/src/test/resources/eml/pgpSignedMail.json b/mailbox/store/src/test/resources/eml/pgpSignedMail.json index e4b004f0111..91bdc424dda 100644 --- a/mailbox/store/src/test/resources/eml/pgpSignedMail.json +++ b/mailbox/store/src/test/resources/eml/pgpSignedMail.json @@ -1,5 +1,6 @@ { - "id": 25, + "id":"184", + "uid":25, "mailboxId": "18", "modSeq": 42, "size": 25, diff --git a/mailbox/store/src/test/resources/eml/recursiveMail.json b/mailbox/store/src/test/resources/eml/recursiveMail.json index 2aa82080bb5..4ec99422445 100644 --- a/mailbox/store/src/test/resources/eml/recursiveMail.json +++ b/mailbox/store/src/test/resources/eml/recursiveMail.json @@ -1,5 +1,6 @@ { - "id": 25, + "id":"184", + "uid":25, "mailboxId": "18", "modSeq": 42, "size": 25, diff --git a/mailbox/store/src/test/resources/eml/recursiveMailWithoutAttachments.json b/mailbox/store/src/test/resources/eml/recursiveMailWithoutAttachments.json index c17741eab6c..06935fdd2f9 100644 --- a/mailbox/store/src/test/resources/eml/recursiveMailWithoutAttachments.json +++ b/mailbox/store/src/test/resources/eml/recursiveMailWithoutAttachments.json @@ -1,5 +1,6 @@ { - "id": 25, + "id":"184", + "uid":25, "mailboxId": "18", "modSeq": 42, "size": 25, diff --git a/mailbox/store/src/test/resources/eml/spamMail.json b/mailbox/store/src/test/resources/eml/spamMail.json index 6e150b02e11..d8ede1e73e8 100644 --- a/mailbox/store/src/test/resources/eml/spamMail.json +++ b/mailbox/store/src/test/resources/eml/spamMail.json @@ -1,5 +1,6 @@ { - "id": 25, + "id":"184", + "uid":25, "mailboxId": "18", "modSeq": 42, "size": 25,