From 3c6ff2c69b0973b18aa0bd8c998c75b778d4a359 Mon Sep 17 00:00:00 2001 From: benwa Date: Fri, 7 Jul 2017 10:37:57 +0700 Subject: [PATCH] JAMES-2092 Cache messages on events whenever possible --- .../cassandra/CassandraMailboxManager.java | 3 +- .../cassandra/CassandraMessageManager.java | 5 +- ...tributedMailboxDelegatingListenerTest.java | 12 +- .../mailbox/hbase/HBaseMailboxManager.java | 3 +- .../mailbox/hbase/HBaseMessageManager.java | 5 +- .../james/mailbox/jcr/JCRMailboxManager.java | 3 +- .../james/mailbox/jcr/JCRMessageManager.java | 5 +- .../james/mailbox/jpa/JPAMessageManager.java | 5 +- .../jpa/openjpa/OpenJPAMailboxManager.java | 3 +- .../jpa/openjpa/OpenJPAMessageManager.java | 9 +- .../inmemory/InMemoryMailboxManager.java | 3 +- .../inmemory/InMemoryMessageManager.java | 5 +- mailbox/store/pom.xml | 1 - .../store/ImmutableMailboxMessage.java | 261 ++++++++++++++++++ .../mailbox/store/StoreMailboxManager.java | 9 +- .../mailbox/store/StoreMessageIdManager.java | 7 +- .../mailbox/store/StoreMessageManager.java | 44 ++- .../mailbox/store/event/EventFactory.java | 13 +- .../store/event/MailboxEventDispatcher.java | 18 +- .../store/json/event/EventConverter.java | 6 +- .../search/ListeningMessageSearchIndex.java | 43 ++- ...bstractMessageIdManagerSideEffectTest.java | 23 +- ...egatingMailboxListenerIntegrationTest.java | 10 +- .../store/json/EventSerializerTest.java | 4 +- .../MailboxRegistrationTest.java | 4 +- .../base/SelectedMailboxImplTest.java | 4 +- .../apache/james/util/IteratorWrapper.java | 60 ++++ .../james/util/IteratorWrapperTest.java | 90 ++++++ 28 files changed, 577 insertions(+), 81 deletions(-) create mode 100644 mailbox/store/src/main/java/org/apache/james/mailbox/store/ImmutableMailboxMessage.java create mode 100644 server/container/util/src/main/java/org/apache/james/util/IteratorWrapper.java create mode 100644 server/container/util/src/test/java/org/apache/james/util/IteratorWrapperTest.java diff --git a/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/CassandraMailboxManager.java b/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/CassandraMailboxManager.java index 3656690f5cb..736ef6a7598 100644 --- a/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/CassandraMailboxManager.java +++ b/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/CassandraMailboxManager.java @@ -133,7 +133,8 @@ protected StoreMessageManager createMessageManager(Mailbox mailboxRow, MailboxSe getQuotaRootResolver(), getMessageParser(), getMessageIdFactory(), - getBatchSizes()); + getBatchSizes(), + getImmutableMailboxMessageFactory()); } } diff --git a/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/CassandraMessageManager.java b/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/CassandraMessageManager.java index 6d5c0855bf5..585e6e35102 100644 --- a/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/CassandraMessageManager.java +++ b/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/CassandraMessageManager.java @@ -30,6 +30,7 @@ import org.apache.james.mailbox.quota.QuotaManager; import org.apache.james.mailbox.quota.QuotaRootResolver; import org.apache.james.mailbox.store.BatchSizes; +import org.apache.james.mailbox.store.ImmutableMailboxMessage; import org.apache.james.mailbox.store.MailboxSessionMapperFactory; import org.apache.james.mailbox.store.StoreMessageManager; import org.apache.james.mailbox.store.event.MailboxEventDispatcher; @@ -46,9 +47,9 @@ public class CassandraMessageManager extends StoreMessageManager { public CassandraMessageManager(MailboxSessionMapperFactory mapperFactory, MessageSearchIndex index, MailboxEventDispatcher dispatcher, MailboxPathLocker locker, Mailbox mailbox, QuotaManager quotaManager, QuotaRootResolver quotaRootResolver, MessageParser messageParser, MessageId.Factory messageIdFactory, - BatchSizes batchSizes) throws MailboxException { + BatchSizes batchSizes, ImmutableMailboxMessage.Factory immutableMailboxMessageFactory) throws MailboxException { super(mapperFactory, index, dispatcher, locker, mailbox, new UnionMailboxACLResolver(), - new SimpleGroupMembershipResolver(), quotaManager, quotaRootResolver, messageParser, messageIdFactory, batchSizes); + new SimpleGroupMembershipResolver(), quotaManager, quotaRootResolver, messageParser, messageIdFactory, batchSizes, immutableMailboxMessageFactory); } diff --git a/mailbox/cassandra/src/test/java/org/apache/james/mailbox/cassandra/event/distributed/CassandraBasedRegisteredDistributedMailboxDelegatingListenerTest.java b/mailbox/cassandra/src/test/java/org/apache/james/mailbox/cassandra/event/distributed/CassandraBasedRegisteredDistributedMailboxDelegatingListenerTest.java index efefb22ebec..37cb6ede1ca 100644 --- a/mailbox/cassandra/src/test/java/org/apache/james/mailbox/cassandra/event/distributed/CassandraBasedRegisteredDistributedMailboxDelegatingListenerTest.java +++ b/mailbox/cassandra/src/test/java/org/apache/james/mailbox/cassandra/event/distributed/CassandraBasedRegisteredDistributedMailboxDelegatingListenerTest.java @@ -27,9 +27,11 @@ import org.apache.james.backends.cassandra.utils.CassandraUtils; import org.apache.james.mailbox.MailboxListener; import org.apache.james.mailbox.MailboxSession; +import org.apache.james.mailbox.MessageUid; import org.apache.james.mailbox.cassandra.modules.CassandraRegistrationModule; import org.apache.james.mailbox.mock.MockMailboxSession; import org.apache.james.mailbox.model.MailboxPath; +import org.apache.james.mailbox.model.MessageMetaData; import org.apache.james.mailbox.model.TestId; import org.apache.james.mailbox.model.TestMessageId; import org.apache.james.mailbox.store.TestIdDeserializer; @@ -40,12 +42,15 @@ import org.apache.james.mailbox.store.json.MessagePackEventSerializer; import org.apache.james.mailbox.store.json.event.EventConverter; import org.apache.james.mailbox.store.json.event.MailboxConverter; +import org.apache.james.mailbox.store.mail.model.MailboxMessage; import org.apache.james.mailbox.store.mail.model.impl.SimpleMailbox; import org.apache.james.mailbox.util.EventCollector; import org.junit.After; import org.junit.Before; import org.junit.Test; +import com.google.common.collect.ImmutableMap; + /** Integration tests for RegisteredDelegatingMailboxListener using a cassandra back-end. @@ -57,6 +62,7 @@ public class CassandraBasedRegisteredDistributedMailboxDelegatingListenerTest { public static final MailboxPath MAILBOX_PATH_2 = new MailboxPath("#private", "user", "mbx.other"); public static final int CASSANDRA_TIME_OUT_IN_S = 10; public static final int SCHEDULER_PERIOD_IN_S = 20; + public static final ImmutableMap EMPTY_MESSAGE_CACHE = ImmutableMap.of(); private CassandraCluster cassandra = CassandraCluster.create(new CassandraRegistrationModule()); private RegisteredDelegatingMailboxListener registeredDelegatingMailboxListener1; @@ -142,7 +148,8 @@ public void tearDown() { public void mailboxEventListenersShouldBeTriggeredIfRegistered() throws Exception { SimpleMailbox simpleMailbox = new SimpleMailbox(MAILBOX_PATH_1, 42); simpleMailbox.setMailboxId(TestId.of(52)); - final MailboxListener.Event event = new EventFactory().added(mailboxSession, new TreeMap<>(), simpleMailbox); + TreeMap uids = new TreeMap<>(); + final MailboxListener.Event event = new EventFactory().added(mailboxSession, uids, simpleMailbox, EMPTY_MESSAGE_CACHE); registeredDelegatingMailboxListener1.event(event); @@ -155,7 +162,8 @@ public void mailboxEventListenersShouldBeTriggeredIfRegistered() throws Exceptio public void onceEventListenersShouldBeTriggeredOnceAcrossTheCluster() { SimpleMailbox simpleMailbox = new SimpleMailbox(MAILBOX_PATH_1, 42); simpleMailbox.setMailboxId(TestId.of(52)); - final MailboxListener.Event event = new EventFactory().added(mailboxSession, new TreeMap<>(), simpleMailbox); + TreeMap uids = new TreeMap<>(); + final MailboxListener.Event event = new EventFactory().added(mailboxSession, uids, simpleMailbox, EMPTY_MESSAGE_CACHE); registeredDelegatingMailboxListener1.event(event); diff --git a/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/HBaseMailboxManager.java b/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/HBaseMailboxManager.java index baeadb93b52..1540b7b6c64 100644 --- a/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/HBaseMailboxManager.java +++ b/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/HBaseMailboxManager.java @@ -105,6 +105,7 @@ protected StoreMessageManager createMessageManager(Mailbox mailboxRow, MailboxSe getQuotaRootResolver(), getMessageParser(), getMessageIdFactory(), - getBatchSizes()); + getBatchSizes(), + getImmutableMailboxMessageFactory()); } } diff --git a/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/HBaseMessageManager.java b/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/HBaseMessageManager.java index 00386b8185f..0b1c05437c4 100644 --- a/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/HBaseMessageManager.java +++ b/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/HBaseMessageManager.java @@ -29,6 +29,7 @@ import org.apache.james.mailbox.quota.QuotaManager; import org.apache.james.mailbox.quota.QuotaRootResolver; import org.apache.james.mailbox.store.BatchSizes; +import org.apache.james.mailbox.store.ImmutableMailboxMessage; import org.apache.james.mailbox.store.MailboxSessionMapperFactory; import org.apache.james.mailbox.store.StoreMessageManager; import org.apache.james.mailbox.store.event.MailboxEventDispatcher; @@ -45,9 +46,9 @@ public class HBaseMessageManager extends StoreMessageManager { public HBaseMessageManager(MailboxSessionMapperFactory mapperFactory, MessageSearchIndex index, MailboxEventDispatcher dispatcher, MailboxPathLocker locker, Mailbox mailbox, MailboxACLResolver aclResolver, GroupMembershipResolver groupMembershipResolver, QuotaManager quotaManager, - QuotaRootResolver quotaRootResolver, MessageParser messageParser, MessageId.Factory messageIdFactory, BatchSizes batchSizes) throws MailboxException { + QuotaRootResolver quotaRootResolver, MessageParser messageParser, MessageId.Factory messageIdFactory, BatchSizes batchSizes, ImmutableMailboxMessage.Factory immutableMailboxMessageFactory) throws MailboxException { super(mapperFactory, index, dispatcher, locker, mailbox, aclResolver, groupMembershipResolver, quotaManager, - quotaRootResolver, messageParser, messageIdFactory, batchSizes); + quotaRootResolver, messageParser, messageIdFactory, batchSizes, immutableMailboxMessageFactory); } diff --git a/mailbox/jcr/src/main/java/org/apache/james/mailbox/jcr/JCRMailboxManager.java b/mailbox/jcr/src/main/java/org/apache/james/mailbox/jcr/JCRMailboxManager.java index c3105b7a3f9..e60ba33e409 100644 --- a/mailbox/jcr/src/main/java/org/apache/james/mailbox/jcr/JCRMailboxManager.java +++ b/mailbox/jcr/src/main/java/org/apache/james/mailbox/jcr/JCRMailboxManager.java @@ -77,7 +77,8 @@ protected StoreMessageManager createMessageManager(Mailbox mailboxEntity, Mailbo getQuotaRootResolver(), getMessageParser(), getMessageIdFactory(), - getBatchSizes()); + getBatchSizes(), + getImmutableMailboxMessageFactory()); } @Override diff --git a/mailbox/jcr/src/main/java/org/apache/james/mailbox/jcr/JCRMessageManager.java b/mailbox/jcr/src/main/java/org/apache/james/mailbox/jcr/JCRMessageManager.java index fbf25e0f1a7..ee76bab85bf 100644 --- a/mailbox/jcr/src/main/java/org/apache/james/mailbox/jcr/JCRMessageManager.java +++ b/mailbox/jcr/src/main/java/org/apache/james/mailbox/jcr/JCRMessageManager.java @@ -36,6 +36,7 @@ import org.apache.james.mailbox.quota.QuotaManager; import org.apache.james.mailbox.quota.QuotaRootResolver; import org.apache.james.mailbox.store.BatchSizes; +import org.apache.james.mailbox.store.ImmutableMailboxMessage; import org.apache.james.mailbox.store.MailboxSessionMapperFactory; import org.apache.james.mailbox.store.StoreMessageManager; import org.apache.james.mailbox.store.event.MailboxEventDispatcher; @@ -56,10 +57,10 @@ public class JCRMessageManager extends StoreMessageManager { public JCRMessageManager(MailboxSessionMapperFactory mapperFactory, MessageSearchIndex index, final MailboxEventDispatcher dispatcher, MailboxPathLocker locker, JCRMailbox mailbox, MailboxACLResolver aclResolver, GroupMembershipResolver groupMembershipResolver, Logger log, - QuotaManager quotaManager, QuotaRootResolver quotaRootResolver, MessageParser messageParser, MessageId.Factory messageIdFactory, BatchSizes batchSizes) + QuotaManager quotaManager, QuotaRootResolver quotaRootResolver, MessageParser messageParser, MessageId.Factory messageIdFactory, BatchSizes batchSizes, ImmutableMailboxMessage.Factory immutableMailboxMessageFactory) throws MailboxException { super(mapperFactory, index, dispatcher, locker, mailbox, aclResolver, groupMembershipResolver, quotaManager, - quotaRootResolver, messageParser, messageIdFactory, batchSizes); + quotaRootResolver, messageParser, messageIdFactory, batchSizes, immutableMailboxMessageFactory); this.log = log; } diff --git a/mailbox/jpa/src/main/java/org/apache/james/mailbox/jpa/JPAMessageManager.java b/mailbox/jpa/src/main/java/org/apache/james/mailbox/jpa/JPAMessageManager.java index 2817940582a..ee023ac794f 100644 --- a/mailbox/jpa/src/main/java/org/apache/james/mailbox/jpa/JPAMessageManager.java +++ b/mailbox/jpa/src/main/java/org/apache/james/mailbox/jpa/JPAMessageManager.java @@ -36,6 +36,7 @@ import org.apache.james.mailbox.quota.QuotaManager; import org.apache.james.mailbox.quota.QuotaRootResolver; import org.apache.james.mailbox.store.BatchSizes; +import org.apache.james.mailbox.store.ImmutableMailboxMessage; import org.apache.james.mailbox.store.MailboxSessionMapperFactory; import org.apache.james.mailbox.store.StoreMessageManager; import org.apache.james.mailbox.store.event.MailboxEventDispatcher; @@ -54,10 +55,10 @@ public JPAMessageManager(MailboxSessionMapperFactory mapperFactory, MessageSearc final MailboxEventDispatcher dispatcher, MailboxPathLocker locker, final Mailbox mailbox, MailboxACLResolver aclResolver, GroupMembershipResolver groupMembershipResolver, QuotaManager quotaManager, - QuotaRootResolver quotaRootResolver, MessageParser messageParser, MessageId.Factory messageIdFactory, BatchSizes batchSizes) throws MailboxException { + QuotaRootResolver quotaRootResolver, MessageParser messageParser, MessageId.Factory messageIdFactory, BatchSizes batchSizes, ImmutableMailboxMessage.Factory immutableMailboxMessageFactory) throws MailboxException { super(mapperFactory, index, dispatcher, locker, mailbox, aclResolver, groupMembershipResolver, - quotaManager, quotaRootResolver, messageParser, messageIdFactory, batchSizes); + quotaManager, quotaRootResolver, messageParser, messageIdFactory, batchSizes, immutableMailboxMessageFactory); } @Override diff --git a/mailbox/jpa/src/main/java/org/apache/james/mailbox/jpa/openjpa/OpenJPAMailboxManager.java b/mailbox/jpa/src/main/java/org/apache/james/mailbox/jpa/openjpa/OpenJPAMailboxManager.java index d6bb542fbab..7944ec49517 100644 --- a/mailbox/jpa/src/main/java/org/apache/james/mailbox/jpa/openjpa/OpenJPAMailboxManager.java +++ b/mailbox/jpa/src/main/java/org/apache/james/mailbox/jpa/openjpa/OpenJPAMailboxManager.java @@ -105,6 +105,7 @@ protected StoreMessageManager createMessageManager(Mailbox mailboxRow, MailboxSe getQuotaRootResolver(), getMessageParser(), getMessageIdFactory(), - getBatchSizes()); + getBatchSizes(), + getImmutableMailboxMessageFactory()); } } diff --git a/mailbox/jpa/src/main/java/org/apache/james/mailbox/jpa/openjpa/OpenJPAMessageManager.java b/mailbox/jpa/src/main/java/org/apache/james/mailbox/jpa/openjpa/OpenJPAMessageManager.java index 7091d36e6ab..ca545d51b94 100644 --- a/mailbox/jpa/src/main/java/org/apache/james/mailbox/jpa/openjpa/OpenJPAMessageManager.java +++ b/mailbox/jpa/src/main/java/org/apache/james/mailbox/jpa/openjpa/OpenJPAMessageManager.java @@ -38,6 +38,7 @@ import org.apache.james.mailbox.quota.QuotaManager; import org.apache.james.mailbox.quota.QuotaRootResolver; import org.apache.james.mailbox.store.BatchSizes; +import org.apache.james.mailbox.store.ImmutableMailboxMessage; import org.apache.james.mailbox.store.MailboxSessionMapperFactory; import org.apache.james.mailbox.store.event.MailboxEventDispatcher; import org.apache.james.mailbox.store.mail.model.Mailbox; @@ -64,9 +65,9 @@ public OpenJPAMessageManager(MailboxSessionMapperFactory mapperFactory, MailboxPathLocker locker, Mailbox mailbox, MailboxACLResolver aclResolver, GroupMembershipResolver groupMembershipResolver, QuotaManager quotaManager, QuotaRootResolver quotaRootResolver, MessageParser messageParser, - MessageId.Factory messageIdFactory, BatchSizes batchSizes) throws MailboxException { + MessageId.Factory messageIdFactory, BatchSizes batchSizes, ImmutableMailboxMessage.Factory immutableMailboxMessageFactory) throws MailboxException { this(mapperFactory, index, dispatcher, locker, mailbox, AdvancedFeature.None, aclResolver, - groupMembershipResolver, quotaManager, quotaRootResolver, messageParser, messageIdFactory, batchSizes); + groupMembershipResolver, quotaManager, quotaRootResolver, messageParser, messageIdFactory, batchSizes, immutableMailboxMessageFactory); } public OpenJPAMessageManager(MailboxSessionMapperFactory mapperFactory, @@ -74,9 +75,9 @@ public OpenJPAMessageManager(MailboxSessionMapperFactory mapperFactory, MailboxPathLocker locker, Mailbox mailbox, AdvancedFeature f, MailboxACLResolver aclResolver, GroupMembershipResolver groupMembershipResolver, QuotaManager quotaManager, QuotaRootResolver quotaRootResolver, MessageParser messageParser, - MessageId.Factory messageIdFactory, BatchSizes batchSizes) throws MailboxException { + MessageId.Factory messageIdFactory, BatchSizes batchSizes, ImmutableMailboxMessage.Factory immutableMailboxMessageFactory) throws MailboxException { - super(mapperFactory, index, dispatcher, locker, mailbox, aclResolver, groupMembershipResolver, quotaManager, quotaRootResolver, messageParser, messageIdFactory, batchSizes); + super(mapperFactory, index, dispatcher, locker, mailbox, aclResolver, groupMembershipResolver, quotaManager, quotaRootResolver, messageParser, messageIdFactory, batchSizes, immutableMailboxMessageFactory); this.feature = f; } diff --git a/mailbox/memory/src/main/java/org/apache/james/mailbox/inmemory/InMemoryMailboxManager.java b/mailbox/memory/src/main/java/org/apache/james/mailbox/inmemory/InMemoryMailboxManager.java index 3b465ef73aa..55584d86c14 100644 --- a/mailbox/memory/src/main/java/org/apache/james/mailbox/inmemory/InMemoryMailboxManager.java +++ b/mailbox/memory/src/main/java/org/apache/james/mailbox/inmemory/InMemoryMailboxManager.java @@ -87,6 +87,7 @@ protected StoreMessageManager createMessageManager(Mailbox mailbox, MailboxSessi getQuotaRootResolver(), getMessageParser(), getMessageIdFactory(), - getBatchSizes()); + getBatchSizes(), + getImmutableMailboxMessageFactory()); } } diff --git a/mailbox/memory/src/main/java/org/apache/james/mailbox/inmemory/InMemoryMessageManager.java b/mailbox/memory/src/main/java/org/apache/james/mailbox/inmemory/InMemoryMessageManager.java index b573b9ba12d..161bd5c981e 100644 --- a/mailbox/memory/src/main/java/org/apache/james/mailbox/inmemory/InMemoryMessageManager.java +++ b/mailbox/memory/src/main/java/org/apache/james/mailbox/inmemory/InMemoryMessageManager.java @@ -11,6 +11,7 @@ import org.apache.james.mailbox.quota.QuotaManager; import org.apache.james.mailbox.quota.QuotaRootResolver; import org.apache.james.mailbox.store.BatchSizes; +import org.apache.james.mailbox.store.ImmutableMailboxMessage; import org.apache.james.mailbox.store.MailboxSessionMapperFactory; import org.apache.james.mailbox.store.StoreMessageManager; import org.apache.james.mailbox.store.event.MailboxEventDispatcher; @@ -22,8 +23,8 @@ public class InMemoryMessageManager extends StoreMessageManager { public InMemoryMessageManager(MailboxSessionMapperFactory mapperFactory, MessageSearchIndex index, MailboxEventDispatcher dispatcher, MailboxPathLocker locker, Mailbox mailbox, MailboxACLResolver aclResolver, GroupMembershipResolver groupMembershipResolver, - QuotaManager quotaManager, QuotaRootResolver quotaRootResolver, MessageParser messageParser, MessageId.Factory messageIdFactory, BatchSizes batchSizes) throws MailboxException { - super(mapperFactory, index, dispatcher, locker, mailbox, aclResolver, groupMembershipResolver, quotaManager, quotaRootResolver, messageParser, messageIdFactory, batchSizes); + QuotaManager quotaManager, QuotaRootResolver quotaRootResolver, MessageParser messageParser, MessageId.Factory messageIdFactory, BatchSizes batchSizes, ImmutableMailboxMessage.Factory immutableMailboxMessageFactory) throws MailboxException { + super(mapperFactory, index, dispatcher, locker, mailbox, aclResolver, groupMembershipResolver, quotaManager, quotaRootResolver, messageParser, messageIdFactory, batchSizes, immutableMailboxMessageFactory); } @Override diff --git a/mailbox/store/pom.xml b/mailbox/store/pom.xml index 7cb4e31007e..95af179dd42 100644 --- a/mailbox/store/pom.xml +++ b/mailbox/store/pom.xml @@ -39,7 +39,6 @@ org.apache.james james-server-util - test org.apache.james diff --git a/mailbox/store/src/main/java/org/apache/james/mailbox/store/ImmutableMailboxMessage.java b/mailbox/store/src/main/java/org/apache/james/mailbox/store/ImmutableMailboxMessage.java new file mode 100644 index 00000000000..1f2b4c5ae01 --- /dev/null +++ b/mailbox/store/src/main/java/org/apache/james/mailbox/store/ImmutableMailboxMessage.java @@ -0,0 +1,261 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.mailbox.store; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Date; +import java.util.List; + +import javax.mail.Flags; +import javax.mail.util.SharedByteArrayInputStream; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.NotImplementedException; +import org.apache.james.mailbox.MailboxManager; +import org.apache.james.mailbox.MessageUid; +import org.apache.james.mailbox.exception.MailboxException; +import org.apache.james.mailbox.model.MailboxId; +import org.apache.james.mailbox.model.MessageAttachment; +import org.apache.james.mailbox.model.MessageId; +import org.apache.james.mailbox.store.mail.model.FlagsBuilder; +import org.apache.james.mailbox.store.mail.model.MailboxMessage; +import org.apache.james.mailbox.store.mail.model.Property; + +import com.google.common.collect.ImmutableList; + +public class ImmutableMailboxMessage implements MailboxMessage { + + public static class Factory { + + private final MailboxManager mailboxManager; + + public Factory(MailboxManager mailboxManager) { + this.mailboxManager = mailboxManager; + } + + public ImmutableMailboxMessage from(MailboxId mailboxId, MailboxMessage message) throws MailboxException { + try { + return new ImmutableMailboxMessage(message.getMessageId(), + message.getInternalDate(), + copy(message.getBodyContent()), + message.getMediaType(), + message.getSubType(), + message.getBodyOctets(), + message.getFullContentOctets(), + message.getFullContentOctets() - message.getBodyOctets(), + message.getTextualLineCount(), + copy(message.getHeaderContent()), + copy(message.getFullContent()), + ImmutableList.copyOf(message.getProperties()), + attachments(message), + mailboxId, + message.getUid(), + message.getModSeq(), + message.isAnswered(), + message.isDeleted(), + message.isDraft(), + message.isFlagged(), + message.isRecent(), + message.isSeen(), + message.createFlags().getUserFlags()); + } catch (IOException e) { + throw new MailboxException("Unable to parse message", e); + } + } + + private ImmutableList attachments(MailboxMessage message) { + if (mailboxManager.getSupportedMessageCapabilities().contains(MailboxManager.MessageCapabilities.Attachment)) { + return ImmutableList.copyOf(message.getAttachments()); + } + return ImmutableList.of(); + } + + private static SharedByteArrayInputStream copy(InputStream inputStream) throws IOException { + return new SharedByteArrayInputStream(IOUtils.toByteArray(inputStream)); + } + } + + private final MessageId messageId; + private final Date internalDate; + private final InputStream bodyContent; + private final String mediaType; + private final String subType; + private final long bodyOctets; + private final long fullContentOctets; + private final long headerOctets; + private final Long textualLineCount; + private final InputStream headerContent; + private final InputStream fullContent; + private final List properties; + private final List attachments; + private final MailboxId mailboxId; + private final MessageUid uid; + private final long modSeq; + private final boolean answered; + private final boolean deleted; + private final boolean draft; + private final boolean flagged; + private final boolean recent; + private final boolean seen; + private final String[] userFlags; + + private ImmutableMailboxMessage(MessageId messageId, Date internalDate, InputStream bodyContent, String mediaType, String subType, long bodyOctets, long fullContentOctets, long headerOctets, Long textualLineCount, InputStream headerContent, + InputStream fullContent, List properties, List attachments, MailboxId mailboxId, MessageUid uid, long modSeq, boolean answered, boolean deleted, boolean draft, boolean flagged, boolean recent, + boolean seen, String[] userFlags) { + this.messageId = messageId; + this.internalDate = internalDate; + this.bodyContent = bodyContent; + this.mediaType = mediaType; + this.subType = subType; + this.bodyOctets = bodyOctets; + this.fullContentOctets = fullContentOctets; + this.headerOctets = headerOctets; + this.textualLineCount = textualLineCount; + this.headerContent = headerContent; + this.fullContent = fullContent; + this.properties = properties; + this.attachments = attachments; + this.mailboxId = mailboxId; + this.uid = uid; + this.modSeq = modSeq; + this.answered = answered; + this.deleted = deleted; + this.draft = draft; + this.flagged = flagged; + this.recent = recent; + this.seen = seen; + this.userFlags = userFlags; + } + + public MessageId getMessageId() { + return messageId; + } + + public Date getInternalDate() { + return internalDate; + } + + public InputStream getBodyContent() { + return bodyContent; + } + + public String getMediaType() { + return mediaType; + } + + public String getSubType() { + return subType; + } + + public long getBodyOctets() { + return bodyOctets; + } + + public long getFullContentOctets() { + return fullContentOctets; + } + + @Override + public long getHeaderOctets() { + return headerOctets; + } + + public Long getTextualLineCount() { + return textualLineCount; + } + + public InputStream getHeaderContent() { + return headerContent; + } + + public InputStream getFullContent() { + return fullContent; + } + + public List getProperties() { + return properties; + } + + public List getAttachments() { + return attachments; + } + + public MailboxId getMailboxId() { + return mailboxId; + } + + public MessageUid getUid() { + return uid; + } + + public long getModSeq() { + return modSeq; + } + + public boolean isAnswered() { + return answered; + } + + public boolean isDeleted() { + return deleted; + } + + public boolean isDraft() { + return draft; + } + + public boolean isFlagged() { + return flagged; + } + + public boolean isRecent() { + return recent; + } + + public boolean isSeen() { + return seen; + } + + @Override + public int compareTo(MailboxMessage o) { + throw new NotImplementedException(); + } + + @Override + public void setUid(MessageUid uid) { + throw new NotImplementedException(); + } + + @Override + public void setModSeq(long modSeq) { + throw new NotImplementedException(); + } + + @Override + public void setFlags(Flags flags) { + throw new NotImplementedException(); + } + + @Override + public Flags createFlags() { + return FlagsBuilder.createFlags(this, userFlags); + } + +} diff --git a/mailbox/store/src/main/java/org/apache/james/mailbox/store/StoreMailboxManager.java b/mailbox/store/src/main/java/org/apache/james/mailbox/store/StoreMailboxManager.java index 1bc28df30b2..355070b2a7a 100644 --- a/mailbox/store/src/main/java/org/apache/james/mailbox/store/StoreMailboxManager.java +++ b/mailbox/store/src/main/java/org/apache/james/mailbox/store/StoreMailboxManager.java @@ -140,6 +140,8 @@ public class StoreMailboxManager implements MailboxManager { private final int limitAnnotationSize; + private final ImmutableMailboxMessage.Factory immutableMailboxMessageFactory; + @Inject public StoreMailboxManager(MailboxSessionMapperFactory mailboxSessionMapperFactory, Authenticator authenticator, Authorizator authorizator, MailboxPathLocker locker, MailboxACLResolver aclResolver, GroupMembershipResolver groupMembershipResolver, @@ -185,6 +187,7 @@ public StoreMailboxManager(MailboxSessionMapperFactory mailboxSessionMapperFacto this.limitAnnotationSize = limitAnnotationSize; this.delegatingListener = delegatingListener; this.dispatcher = mailboxEventDispatcher; + this.immutableMailboxMessageFactory = new ImmutableMailboxMessage.Factory(this); } protected Factory getMessageIdFactory() { @@ -223,6 +226,10 @@ public BatchSizes getBatchSizes() { return batchSizes; } + public ImmutableMailboxMessage.Factory getImmutableMailboxMessageFactory() { + return immutableMailboxMessageFactory; + } + /** * Init the {@link MailboxManager} * @@ -464,7 +471,7 @@ public void logout(MailboxSession session, boolean force) throws MailboxExceptio protected StoreMessageManager createMessageManager(Mailbox mailbox, MailboxSession session) throws MailboxException { return new StoreMessageManager(getMapperFactory(), getMessageSearchIndex(), getEventDispatcher(), getLocker(), mailbox, getAclResolver(), getGroupMembershipResolver(), getQuotaManager(), - getQuotaRootResolver(), getMessageParser(), getMessageIdFactory(), getBatchSizes()); + getQuotaRootResolver(), getMessageParser(), getMessageIdFactory(), getBatchSizes(), getImmutableMailboxMessageFactory()); } /** diff --git a/mailbox/store/src/main/java/org/apache/james/mailbox/store/StoreMessageIdManager.java b/mailbox/store/src/main/java/org/apache/james/mailbox/store/StoreMessageIdManager.java index 9964d6c8817..7c883bbb23f 100644 --- a/mailbox/store/src/main/java/org/apache/james/mailbox/store/StoreMessageIdManager.java +++ b/mailbox/store/src/main/java/org/apache/james/mailbox/store/StoreMessageIdManager.java @@ -229,18 +229,17 @@ private void addMessageToMailboxes(MessageIdMapper messageIdMapper, MailboxMessa MailboxMapper mailboxMapper = mailboxSessionMapperFactory.getMailboxMapper(mailboxSession); for (MailboxId mailboxId : mailboxIds) { SimpleMailboxMessage copy = SimpleMailboxMessage.copy(mailboxId, mailboxMessage); - MessageMetaData metaData = save(mailboxSession, messageIdMapper, copy); - dispatcher.added(mailboxSession, metaData, mailboxMapper.findMailboxById(mailboxId)); + save(mailboxSession, messageIdMapper, copy); + dispatcher.added(mailboxSession, mailboxMapper.findMailboxById(mailboxId), copy); } } - private MessageMetaData save(MailboxSession mailboxSession, MessageIdMapper messageIdMapper, MailboxMessage mailboxMessage) throws MailboxException { + private void save(MailboxSession mailboxSession, MessageIdMapper messageIdMapper, MailboxMessage mailboxMessage) throws MailboxException { long modSeq = mailboxSessionMapperFactory.getModSeqProvider().nextModSeq(mailboxSession, mailboxMessage.getMailboxId()); MessageUid uid = mailboxSessionMapperFactory.getUidProvider().nextUid(mailboxSession, mailboxMessage.getMailboxId()); mailboxMessage.setModSeq(modSeq); mailboxMessage.setUid(uid); messageIdMapper.save(mailboxMessage); - return new SimpleMessageMetaData(uid, modSeq, mailboxMessage.createFlags(), mailboxMessage.getFullContentOctets(), mailboxMessage.getInternalDate(), mailboxMessage.getMessageId()); } private Function messageResultConverter(final MessageResult.FetchGroup fetchGroup) { diff --git a/mailbox/store/src/main/java/org/apache/james/mailbox/store/StoreMessageManager.java b/mailbox/store/src/main/java/org/apache/james/mailbox/store/StoreMessageManager.java index 69233f7f6c3..137433c590f 100644 --- a/mailbox/store/src/main/java/org/apache/james/mailbox/store/StoreMessageManager.java +++ b/mailbox/store/src/main/java/org/apache/james/mailbox/store/StoreMessageManager.java @@ -91,6 +91,7 @@ import org.apache.james.mime4j.stream.MimeConfig; import org.apache.james.mime4j.stream.MimeTokenStream; import org.apache.james.mime4j.stream.RecursionMode; +import org.apache.james.util.IteratorWrapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -98,6 +99,7 @@ import com.google.common.base.Predicate; import com.google.common.collect.FluentIterable; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; /** * Base class for {@link org.apache.james.mailbox.MessageManager} @@ -171,9 +173,12 @@ public boolean apply(MessageAttachment input) { private BatchSizes batchSizes = BatchSizes.defaultValues(); + private final ImmutableMailboxMessage.Factory immutableMailboxMessageFactory; + public StoreMessageManager(MailboxSessionMapperFactory mapperFactory, MessageSearchIndex index, MailboxEventDispatcher dispatcher, MailboxPathLocker locker, Mailbox mailbox, MailboxACLResolver aclResolver, GroupMembershipResolver groupMembershipResolver, - QuotaManager quotaManager, QuotaRootResolver quotaRootResolver, MessageParser messageParser, MessageId.Factory messageIdFactory, BatchSizes batchSizes) + QuotaManager quotaManager, QuotaRootResolver quotaRootResolver, MessageParser messageParser, MessageId.Factory messageIdFactory, BatchSizes batchSizes, + ImmutableMailboxMessage.Factory immutableMailboxMessageFactory) throws MailboxException { this.mailbox = mailbox; this.dispatcher = dispatcher; @@ -187,6 +192,7 @@ public StoreMessageManager(MailboxSessionMapperFactory mapperFactory, MessageSea this.messageParser = messageParser; this.messageIdFactory = messageIdFactory; this.batchSizes = batchSizes; + this.immutableMailboxMessageFactory = immutableMailboxMessageFactory; } protected Factory getMessageIdFactory() { @@ -427,12 +433,9 @@ public ComposedMessageId appendMessage(InputStream msgIn, Date internalDate, fin public ComposedMessageId execute() throws MailboxException { MessageMetaData data = appendMessageToStore(message, attachments, mailboxSession); - SortedMap uids = new TreeMap(); - MessageUid messageUid = data.getUid(); - MailboxId mailboxId = getMailboxEntity().getMailboxId(); - uids.put(messageUid, data); - dispatcher.added(mailboxSession, uids, getMailboxEntity()); - return new ComposedMessageId(mailboxId, data.getMessageId(), messageUid); + Mailbox mailbox = getMailboxEntity(); + dispatcher.added(mailboxSession, mailbox, message); + return new ComposedMessageId(mailbox.getMailboxId(), data.getMessageId(), data.getUid()); } }, true); @@ -632,7 +635,6 @@ public List copyTo(final MessageRange set, final StoreMessageManag @Override public List execute() throws MailboxException { SortedMap copiedUids = copy(set, toMailbox, session); - dispatcher.added(session, copiedUids, toMailbox.getMailboxEntity()); return MessageRange.toRanges(new ArrayList(copiedUids.keySet())); } }, true); @@ -660,7 +662,6 @@ public List moveTo(final MessageRange set, final StoreMessageManag @Override public List execute() throws MailboxException { SortedMap movedUids = move(set, toMailbox, session); - dispatcher.added(session, movedUids, toMailbox.getMailboxEntity()); return MessageRange.toRanges(new ArrayList(movedUids.keySet())); } }, true); @@ -795,15 +796,32 @@ public MessageMetaData run() throws MailboxException { private SortedMap copy(MessageRange set, StoreMessageManager to, MailboxSession session) throws MailboxException { - Iterator originalRows = retrieveOriginalRows(set, session); - return collectMetadata(to.copy(originalRows, session)); + IteratorWrapper originalRows = new IteratorWrapper(retrieveOriginalRows(set, session)); + + SortedMap copiedUids = collectMetadata(to.copy(originalRows, session)); + + ImmutableMap.Builder messagesMap = ImmutableMap.builder(); + for(MailboxMessage message: originalRows.getEntriesSeen()) { + messagesMap.put(message.getUid(), immutableMailboxMessageFactory.from(to.getMailboxEntity().getMailboxId(), message)); + } + dispatcher.added(session, copiedUids, to.getMailboxEntity(), messagesMap.build()); + + return copiedUids; } private SortedMap move(MessageRange set, StoreMessageManager to, MailboxSession session) throws MailboxException { - Iterator originalRows = retrieveOriginalRows(set, session); + IteratorWrapper originalRows = new IteratorWrapper(retrieveOriginalRows(set, session)); + MoveResult moveResult = to.move(originalRows, session); + SortedMap moveUids = collectMetadata(moveResult.getMovedMessages()); + + ImmutableMap.Builder messagesMap = ImmutableMap.builder(); + for(MailboxMessage message: originalRows.getEntriesSeen()) { + messagesMap.put(message.getUid(), immutableMailboxMessageFactory.from(to.getMailboxEntity().getMailboxId(), message)); + } + dispatcher.added(session, moveUids, to.getMailboxEntity(), messagesMap.build()); dispatcher.expunged(session, collectMetadata(moveResult.getOriginalMessages()), getMailboxEntity()); - return collectMetadata(moveResult.getMovedMessages()); + return moveUids; } private Iterator retrieveOriginalRows(MessageRange set, MailboxSession session) throws MailboxException { diff --git a/mailbox/store/src/main/java/org/apache/james/mailbox/store/event/EventFactory.java b/mailbox/store/src/main/java/org/apache/james/mailbox/store/event/EventFactory.java index f2da9e52ece..1cffdc4bef2 100644 --- a/mailbox/store/src/main/java/org/apache/james/mailbox/store/event/EventFactory.java +++ b/mailbox/store/src/main/java/org/apache/james/mailbox/store/event/EventFactory.java @@ -31,6 +31,7 @@ import org.apache.james.mailbox.model.UpdatedFlags; import org.apache.james.mailbox.store.StoreMailboxPath; import org.apache.james.mailbox.store.mail.model.Mailbox; +import org.apache.james.mailbox.store.mail.model.MailboxMessage; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -43,12 +44,14 @@ public interface MailboxAware { public final class AddedImpl extends MailboxListener.Added implements MailboxAware { private final Map added; + private final Map availableMessages; private final Mailbox mailbox; - public AddedImpl(MailboxSession session, Mailbox mailbox, SortedMap uids) { + public AddedImpl(MailboxSession session, Mailbox mailbox, SortedMap uids, Map availableMessages) { super(session, new StoreMailboxPath(mailbox)); this.added = ImmutableMap.copyOf(uids); this.mailbox = mailbox; + this.availableMessages = ImmutableMap.copyOf(availableMessages); } public List getUids() { @@ -62,6 +65,10 @@ public MessageMetaData getMetaData(MessageUid uid) { public Mailbox getMailbox() { return mailbox; } + + public Map getAvailableMessages() { + return availableMessages; + } } public final class ExpungedImpl extends MailboxListener.Expunged implements MailboxAware { @@ -167,8 +174,8 @@ public Mailbox getMailbox() { } } - public MailboxListener.Added added(MailboxSession session, SortedMap uids, Mailbox mailbox) { - return new AddedImpl(session, mailbox, uids); + public MailboxListener.Added added(MailboxSession session, SortedMap uids, Mailbox mailbox, Map cachedMessages) { + return new AddedImpl(session, mailbox, uids, cachedMessages); } public MailboxListener.Expunged expunged(MailboxSession session, Map uids, Mailbox mailbox) { diff --git a/mailbox/store/src/main/java/org/apache/james/mailbox/store/event/MailboxEventDispatcher.java b/mailbox/store/src/main/java/org/apache/james/mailbox/store/event/MailboxEventDispatcher.java index 0d64ed9a4ee..91b79550f07 100644 --- a/mailbox/store/src/main/java/org/apache/james/mailbox/store/event/MailboxEventDispatcher.java +++ b/mailbox/store/src/main/java/org/apache/james/mailbox/store/event/MailboxEventDispatcher.java @@ -31,7 +31,9 @@ import org.apache.james.mailbox.model.MailboxPath; import org.apache.james.mailbox.model.MessageMetaData; import org.apache.james.mailbox.model.UpdatedFlags; +import org.apache.james.mailbox.store.SimpleMessageMetaData; import org.apache.james.mailbox.store.mail.model.Mailbox; +import org.apache.james.mailbox.store.mail.model.MailboxMessage; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; @@ -69,15 +71,23 @@ private MailboxEventDispatcher(MailboxListener listener, EventFactory eventFacto * @param uids Sorted map with uids and message meta data * @param mailbox The mailbox */ - public void added(MailboxSession session, SortedMap uids, Mailbox mailbox) { - listener.event(eventFactory.added(session, uids, mailbox)); + public void added(MailboxSession session, SortedMap uids, Mailbox mailbox, Map cachedMessages) { + listener.event(eventFactory.added(session, uids, mailbox, cachedMessages)); } - public void added(MailboxSession session, MessageMetaData messageMetaData, Mailbox mailbox) { + public void added(MailboxSession session, Mailbox mailbox, MailboxMessage mailboxMessage) { + SimpleMessageMetaData messageMetaData = new SimpleMessageMetaData(mailboxMessage); SortedMap metaDataMap = ImmutableSortedMap.naturalOrder() .put(messageMetaData.getUid(), messageMetaData) .build(); - added(session, metaDataMap, mailbox); + added(session, metaDataMap, mailbox, ImmutableMap.of(mailboxMessage.getUid(), mailboxMessage)); + } + + public void added(MailboxSession session, MessageMetaData messageMetaData, Mailbox mailbox) { + SortedMap metaDataMap = ImmutableSortedMap.naturalOrder() + .put(messageMetaData.getUid(), messageMetaData) + .build(); + added(session, metaDataMap, mailbox, ImmutableMap.of()); } /** diff --git a/mailbox/store/src/main/java/org/apache/james/mailbox/store/json/event/EventConverter.java b/mailbox/store/src/main/java/org/apache/james/mailbox/store/json/event/EventConverter.java index 866c5945f31..0621d55695d 100644 --- a/mailbox/store/src/main/java/org/apache/james/mailbox/store/json/event/EventConverter.java +++ b/mailbox/store/src/main/java/org/apache/james/mailbox/store/json/event/EventConverter.java @@ -42,9 +42,12 @@ import org.apache.james.mailbox.store.json.event.dto.MessageMetaDataDataTransferObject; import org.apache.james.mailbox.store.json.event.dto.UpdatedFlagsDataTransferObject; import org.apache.james.mailbox.store.mail.model.Mailbox; +import org.apache.james.mailbox.store.mail.model.MailboxMessage; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.collect.ImmutableMap; + public class EventConverter { private static final Logger LOG = LoggerFactory.getLogger(EventConverter.class); @@ -98,7 +101,8 @@ public MailboxListener.Event retrieveEvent(EventDataTransferObject eventDataTran case ADDED: return eventFactory.added(eventDataTransferObject.getSession().getMailboxSession(), retrieveMetadata(eventDataTransferObject.getMetaDataProxyMap()), - mailbox); + mailbox, + ImmutableMap.of()); case DELETED: return eventFactory.expunged(eventDataTransferObject.getSession().getMailboxSession(), retrieveMetadata(eventDataTransferObject.getMetaDataProxyMap()), diff --git a/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/ListeningMessageSearchIndex.java b/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/ListeningMessageSearchIndex.java index 11464d3a70e..bd4521c9fb2 100644 --- a/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/ListeningMessageSearchIndex.java +++ b/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/ListeningMessageSearchIndex.java @@ -18,7 +18,6 @@ ****************************************************************/ package org.apache.james.mailbox.store.search; -import java.util.Iterator; import java.util.List; import org.apache.james.mailbox.MailboxListener; @@ -33,6 +32,8 @@ import org.apache.james.mailbox.store.mail.model.Mailbox; import org.apache.james.mailbox.store.mail.model.MailboxMessage; +import com.google.common.base.Optional; + /** * {@link MessageSearchIndex} which needs to get registered as global {@link MailboxListener} and so get * notified about message changes. This will then allow to update the underlying index. @@ -41,6 +42,7 @@ */ public abstract class ListeningMessageSearchIndex implements MessageSearchIndex, MailboxListener { + public static final int UNLIMITED = -1; private final MessageMapperFactory factory; public ListeningMessageSearchIndex(MessageMapperFactory factory) { @@ -76,17 +78,11 @@ public void event(Event event) { EventFactory.AddedImpl added = (EventFactory.AddedImpl) event; final Mailbox mailbox = added.getMailbox(); - for (MessageUid next : (Iterable) added.getUids()) { - Iterator messages = factory.getMessageMapper(session).findInMailbox(mailbox, MessageRange.one(next), FetchType.Full, -1); - while (messages.hasNext()) { - MailboxMessage message = messages.next(); - try { - add(session, mailbox, message); - } catch (MailboxException e) { - session.getLog().error("Unable to index message " + message.getUid() + " for mailbox " + mailbox, e); - } + for (final MessageUid next : (Iterable) added.getUids()) { + Optional mailboxMessage = retrieveMailboxMessage(session, added, mailbox, next); + if (mailboxMessage.isPresent()) { + addMessage(session, mailbox, mailboxMessage.get()); } - } } else if (event instanceof EventFactory.ExpungedImpl) { EventFactory.ExpungedImpl expunged = (EventFactory.ExpungedImpl) event; @@ -113,6 +109,31 @@ public void event(Event event) { } } + private Optional retrieveMailboxMessage(MailboxSession session, EventFactory.AddedImpl added, Mailbox mailbox, MessageUid next) { + Optional firstChoice = Optional.fromNullable(added.getAvailableMessages().get(next)); + if (firstChoice.isPresent()) { + return firstChoice; + } else { + try { + return Optional.of(factory.getMessageMapper(session) + .findInMailbox(mailbox, MessageRange.one(next), FetchType.Full, UNLIMITED) + .next()); + } catch (Exception e) { + session.getLog().error(String.format("Could not retrieve message %d in mailbox %s", + next, mailbox.getMailboxId().serialize()), e); + return Optional.absent(); + } + } + } + + private void addMessage(final MailboxSession session, final Mailbox mailbox, MailboxMessage message) { + try { + add(session, mailbox, message); + } catch (MailboxException e) { + session.getLog().error("Unable to index message " + message.getUid() + " for mailbox " + mailbox, e); + } + } + /** * Add the {@link MailboxMessage} for the given {@link Mailbox} to the index * diff --git a/mailbox/store/src/test/java/org/apache/james/mailbox/store/AbstractMessageIdManagerSideEffectTest.java b/mailbox/store/src/test/java/org/apache/james/mailbox/store/AbstractMessageIdManagerSideEffectTest.java index ec2dac12000..5aa005ed1fc 100644 --- a/mailbox/store/src/test/java/org/apache/james/mailbox/store/AbstractMessageIdManagerSideEffectTest.java +++ b/mailbox/store/src/test/java/org/apache/james/mailbox/store/AbstractMessageIdManagerSideEffectTest.java @@ -30,6 +30,7 @@ import static org.mockito.Mockito.when; import java.util.List; +import java.util.Map; import javax.mail.Flags; @@ -52,6 +53,7 @@ import org.apache.james.mailbox.quota.QuotaManager; import org.apache.james.mailbox.store.event.MailboxEventDispatcher; import org.apache.james.mailbox.store.mail.model.Mailbox; +import org.apache.james.mailbox.store.mail.model.MailboxMessage; import org.apache.james.mailbox.store.quota.QuotaImpl; import org.junit.After; import org.junit.Before; @@ -151,9 +153,8 @@ public void setInMailboxesShouldCallDispatcher() throws Exception { .from(messageIdManager.getMessages(ImmutableList.of(messageId), FetchGroupImpl.MINIMAL, session)) .filter(inMailbox(mailbox1.getMailboxId())) .get(0); - SimpleMessageMetaData simpleMessageMetaData = fromMessageResult(messageId, messageResult); - verify(dispatcher).added(session, simpleMessageMetaData, mailbox1); + verify(dispatcher).added(eq(session), eq(mailbox1), any(MailboxMessage.class)); verifyNoMoreInteractions(dispatcher); } @@ -165,20 +166,10 @@ public void setInMailboxesShouldCallDispatcherWithMultipleMailboxes() throws Exc messageIdManager.setInMailboxes(messageId, ImmutableList.of(mailbox1.getMailboxId(), mailbox2.getMailboxId(), mailbox3.getMailboxId()), session); - List messageResults = messageIdManager.getMessages(ImmutableList.of(messageId), FetchGroupImpl.MINIMAL, session); - MessageResult messageResultMailbox1 = FluentIterable - .from(messageResults) - .filter(inMailbox(mailbox1.getMailboxId())) - .get(0); - SimpleMessageMetaData metadataMailbox1 = fromMessageResult(messageId, messageResultMailbox1); - MessageResult messageResultMailbox3 = FluentIterable - .from(messageResults) - .filter(inMailbox(mailbox3.getMailboxId())) - .get(0); - SimpleMessageMetaData metadataMailbox3 = fromMessageResult(messageId, messageResultMailbox3); + messageIdManager.getMessages(ImmutableList.of(messageId), FetchGroupImpl.MINIMAL, session); - verify(dispatcher).added(session, metadataMailbox1, mailbox1); - verify(dispatcher).added(session, metadataMailbox3, mailbox3); + verify(dispatcher).added(eq(session), eq(mailbox1), any(MailboxMessage.class)); + verify(dispatcher).added(eq(session), eq(mailbox3), any(MailboxMessage.class)); verifyNoMoreInteractions(dispatcher); } @@ -207,7 +198,7 @@ public void setInMailboxesShouldCallDispatchForOnlyAddedAndRemovedMailboxes() th messageIdManager.setInMailboxes(messageId, ImmutableList.of(mailbox1.getMailboxId(), mailbox3.getMailboxId()), session); verify(dispatcher).expunged(eq(session), any(SimpleMessageMetaData.class), eq(mailbox2)); - verify(dispatcher).added(eq(session), any(SimpleMessageMetaData.class), eq(mailbox3)); + verify(dispatcher).added(eq(session), eq(mailbox3), any(MailboxMessage.class)); verifyNoMoreInteractions(dispatcher); } diff --git a/mailbox/store/src/test/java/org/apache/james/mailbox/store/event/distributed/BroadcastDelegatingMailboxListenerIntegrationTest.java b/mailbox/store/src/test/java/org/apache/james/mailbox/store/event/distributed/BroadcastDelegatingMailboxListenerIntegrationTest.java index daf5b2ad90b..e36046828b9 100644 --- a/mailbox/store/src/test/java/org/apache/james/mailbox/store/event/distributed/BroadcastDelegatingMailboxListenerIntegrationTest.java +++ b/mailbox/store/src/test/java/org/apache/james/mailbox/store/event/distributed/BroadcastDelegatingMailboxListenerIntegrationTest.java @@ -36,11 +36,14 @@ import org.apache.james.mailbox.store.json.MessagePackEventSerializer; import org.apache.james.mailbox.store.json.event.EventConverter; import org.apache.james.mailbox.store.json.event.MailboxConverter; +import org.apache.james.mailbox.store.mail.model.MailboxMessage; import org.apache.james.mailbox.store.mail.model.impl.SimpleMailbox; import org.apache.james.mailbox.util.EventCollector; import org.junit.Before; import org.junit.Test; +import com.google.common.collect.ImmutableMap; + /** Integration tests for BroadcastDelegatingMailboxListener. @@ -51,6 +54,7 @@ public class BroadcastDelegatingMailboxListenerIntegrationTest { public static final MailboxPath MAILBOX_PATH_1 = new MailboxPath("#private", "user", "mbx"); public static final MailboxPath MAILBOX_PATH_2 = new MailboxPath("#private", "user", "mbx.other"); public static final String TOPIC = "TOPIC"; + public static final ImmutableMap EMPTY_MESSAGE_CACHE = ImmutableMap.of(); private BroadcastDelegatingMailboxListener broadcastDelegatingMailboxListener1; private BroadcastDelegatingMailboxListener broadcastDelegatingMailboxListener2; private BroadcastDelegatingMailboxListener broadcastDelegatingMailboxListener3; @@ -114,7 +118,7 @@ public void setUp() throws Exception { public void mailboxEventListenersShouldBeTriggeredIfRegistered() throws Exception { SimpleMailbox simpleMailbox = new SimpleMailbox(MAILBOX_PATH_1, 42); simpleMailbox.setMailboxId(TestId.of(52)); - final MailboxListener.Event event = new EventFactory().added(mailboxSession, new TreeMap(), simpleMailbox); + final MailboxListener.Event event = new EventFactory().added(mailboxSession, new TreeMap(), simpleMailbox, EMPTY_MESSAGE_CACHE); broadcastDelegatingMailboxListener1.event(event); @@ -127,7 +131,7 @@ public void mailboxEventListenersShouldBeTriggeredIfRegistered() throws Exceptio public void onceEventListenersShouldBeTriggeredOnceAcrossTheCluster() { SimpleMailbox simpleMailbox = new SimpleMailbox(MAILBOX_PATH_1, 42); simpleMailbox.setMailboxId(TestId.of(52)); - final MailboxListener.Event event = new EventFactory().added(mailboxSession, new TreeMap(), simpleMailbox); + final MailboxListener.Event event = new EventFactory().added(mailboxSession, new TreeMap(), simpleMailbox, EMPTY_MESSAGE_CACHE); broadcastDelegatingMailboxListener1.event(event); @@ -140,7 +144,7 @@ public void onceEventListenersShouldBeTriggeredOnceAcrossTheCluster() { public void eachEventListenersShouldBeTriggeredOnEachNode() { SimpleMailbox simpleMailbox = new SimpleMailbox(MAILBOX_PATH_1, 42); simpleMailbox.setMailboxId(TestId.of(52)); - final MailboxListener.Event event = new EventFactory().added(mailboxSession, new TreeMap(), simpleMailbox); + final MailboxListener.Event event = new EventFactory().added(mailboxSession, new TreeMap(), simpleMailbox, EMPTY_MESSAGE_CACHE); broadcastDelegatingMailboxListener1.event(event); diff --git a/mailbox/store/src/test/java/org/apache/james/mailbox/store/json/EventSerializerTest.java b/mailbox/store/src/test/java/org/apache/james/mailbox/store/json/EventSerializerTest.java index 900225afc79..76c118563df 100644 --- a/mailbox/store/src/test/java/org/apache/james/mailbox/store/json/EventSerializerTest.java +++ b/mailbox/store/src/test/java/org/apache/james/mailbox/store/json/EventSerializerTest.java @@ -38,10 +38,12 @@ import org.apache.james.mailbox.store.SimpleMessageMetaData; import org.apache.james.mailbox.store.event.EventFactory; import org.apache.james.mailbox.store.event.EventSerializer; +import org.apache.james.mailbox.store.mail.model.MailboxMessage; import org.apache.james.mailbox.store.mail.model.impl.SimpleMailbox; import org.junit.Before; import org.junit.Test; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; public abstract class EventSerializerTest { @@ -80,7 +82,7 @@ public void setUp() { public void addedEventShouldBeWellConverted() throws Exception { TreeMap treeMap = new TreeMap(); treeMap.put(UID, MESSAGE_META_DATA); - MailboxListener.Event event = eventFactory.added(mailboxSession, treeMap, mailbox); + MailboxListener.Event event = eventFactory.added(mailboxSession, treeMap, mailbox, ImmutableMap.of()); byte[] serializedEvent = serializer.serializeEvent(event); MailboxListener.Event deserializedEvent = serializer.deSerializeEvent(serializedEvent); assertThat(deserializedEvent.getMailboxPath()).isEqualTo(event.getMailboxPath()); diff --git a/mailbox/tool/src/test/java/org/apache/james/mailbox/indexer/registrations/MailboxRegistrationTest.java b/mailbox/tool/src/test/java/org/apache/james/mailbox/indexer/registrations/MailboxRegistrationTest.java index 21992e29610..480d3f41aa3 100644 --- a/mailbox/tool/src/test/java/org/apache/james/mailbox/indexer/registrations/MailboxRegistrationTest.java +++ b/mailbox/tool/src/test/java/org/apache/james/mailbox/indexer/registrations/MailboxRegistrationTest.java @@ -37,10 +37,12 @@ import org.apache.james.mailbox.store.SimpleMessageMetaData; import org.apache.james.mailbox.store.event.EventFactory; import org.apache.james.mailbox.store.mail.model.DefaultMessageId; +import org.apache.james.mailbox.store.mail.model.MailboxMessage; import org.apache.james.mailbox.store.mail.model.impl.SimpleMailbox; import org.junit.Before; import org.junit.Test; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; public class MailboxRegistrationTest { @@ -73,7 +75,7 @@ public void reportedEventsShouldBeInitiallyEmpty() { public void AddedEventsShouldNotBeReported() { TreeMap treeMap = new TreeMap(); treeMap.put(UID, new SimpleMessageMetaData(UID, MOD_SEQ, new Flags(), SIZE, new Date(), new DefaultMessageId())); - MailboxListener.Event event = eventFactory.added(session, treeMap, MAILBOX); + MailboxListener.Event event = eventFactory.added(session, treeMap, MAILBOX, ImmutableMap. of()); mailboxRegistration.event(event); assertThat(mailboxRegistration.getImpactingEvents(UID)).isEmpty(); } diff --git a/protocols/imap/src/test/java/org/apache/james/imap/processor/base/SelectedMailboxImplTest.java b/protocols/imap/src/test/java/org/apache/james/imap/processor/base/SelectedMailboxImplTest.java index cdeef781091..33c61e47d49 100644 --- a/protocols/imap/src/test/java/org/apache/james/imap/processor/base/SelectedMailboxImplTest.java +++ b/protocols/imap/src/test/java/org/apache/james/imap/processor/base/SelectedMailboxImplTest.java @@ -50,6 +50,7 @@ import org.apache.james.mailbox.store.event.EventFactory; import org.apache.james.mailbox.store.mail.model.DefaultMessageId; import org.apache.james.mailbox.store.mail.model.Mailbox; +import org.apache.james.mailbox.store.mail.model.MailboxMessage; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -59,6 +60,7 @@ import org.slf4j.LoggerFactory; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; public class SelectedMailboxImplTest { @@ -169,6 +171,6 @@ public void run() { private void emitEvent(MailboxListener mailboxListener) { TreeMap result = new TreeMap(); result.put(EMITTED_EVENT_UID, new SimpleMessageMetaData(EMITTED_EVENT_UID, MOD_SEQ, new Flags(), SIZE, new Date(), new DefaultMessageId())); - mailboxListener.event(new EventFactory().added(mock(MailboxSession.class), result, mailbox)); + mailboxListener.event(new EventFactory().added(mock(MailboxSession.class), result, mailbox, ImmutableMap. of())); } } diff --git a/server/container/util/src/main/java/org/apache/james/util/IteratorWrapper.java b/server/container/util/src/main/java/org/apache/james/util/IteratorWrapper.java new file mode 100644 index 00000000000..486e796c0fd --- /dev/null +++ b/server/container/util/src/main/java/org/apache/james/util/IteratorWrapper.java @@ -0,0 +1,60 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james.util; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +public class IteratorWrapper implements Iterator { + + private final Iterator underlying; + private final List entriesSeen; + + public IteratorWrapper(Iterator underlying) { + Preconditions.checkNotNull(underlying); + this.underlying = underlying; + this.entriesSeen = new ArrayList(); + } + + public List getEntriesSeen() { + return ImmutableList.copyOf(entriesSeen); + } + + @Override + public boolean hasNext() { + return underlying.hasNext(); + } + + @Override + public U next() { + U next = underlying.next(); + entriesSeen.add(next); + return next; + } + + @Override + public void remove() { + underlying.remove(); + } +} diff --git a/server/container/util/src/test/java/org/apache/james/util/IteratorWrapperTest.java b/server/container/util/src/test/java/org/apache/james/util/IteratorWrapperTest.java new file mode 100644 index 00000000000..61d7f2deefc --- /dev/null +++ b/server/container/util/src/test/java/org/apache/james/util/IteratorWrapperTest.java @@ -0,0 +1,90 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james.util; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import com.google.common.collect.ImmutableList; + +public class IteratorWrapperTest { + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + @Test + public void getEntriesSeenShouldReturnEmptyWhenNotConsumed() { + ImmutableList originalData = ImmutableList.of(1, 2, 3); + IteratorWrapper integerIteratorWrapper = new IteratorWrapper(originalData.iterator()); + + assertThat(integerIteratorWrapper.getEntriesSeen()).isEmpty(); + } + + @Test + public void getEntriesSeenShouldReturnViewOfConsumedData() { + ImmutableList originalData = ImmutableList.of(1, 2, 3); + IteratorWrapper integerIteratorWrapper = new IteratorWrapper(originalData.iterator()); + + consume(integerIteratorWrapper); + + assertThat(integerIteratorWrapper.getEntriesSeen()) + .containsExactlyElementsOf(originalData); + } + + @Test + public void getEntriesSeenShouldReturnViewOfConsumedDataWhenPartiallyConsumed() { + ImmutableList originalData = ImmutableList.of(1, 2, 3); + IteratorWrapper integerIteratorWrapper = new IteratorWrapper(originalData.iterator()); + + integerIteratorWrapper.next(); + integerIteratorWrapper.next(); + + assertThat(integerIteratorWrapper.getEntriesSeen()) + .containsOnly(1, 2); + } + + @Test + public void getEntriesSeenShouldReturnEmptyWhenSuppliedEmpty() { + ImmutableList originalData = ImmutableList.of(); + IteratorWrapper integerIteratorWrapper = new IteratorWrapper(originalData.iterator()); + + consume(integerIteratorWrapper); + + assertThat(integerIteratorWrapper.getEntriesSeen()) + .containsExactlyElementsOf(originalData); + } + + @Test + public void constructorShouldThrowOnNull() { + expectedException.expect(NullPointerException.class); + + new IteratorWrapper(null); + } + + private void consume(IteratorWrapper integerIteratorWrapper) { + while (integerIteratorWrapper.hasNext()) { + integerIteratorWrapper.next(); + } + } + +}