From 3ae482a711c5a6b5310713553adfbb6102298ff9 Mon Sep 17 00:00:00 2001 From: benwa Date: Thu, 17 May 2018 16:17:58 +0700 Subject: [PATCH] JAMES-2398 Modify injections and configuration to allow several indexes, with different mappings, pointed by different aliases. --- .../backends/es/DeleteByQueryPerformer.java | 9 +- .../backends/es/ElasticSearchConstants.java | 27 ----- ...Indexer.java => ElasticSearchIndexer.java} | 23 +++- .../james/backends/es/IndexerSupplier.java | 24 ---- ...est.java => ElasticSearchIndexerTest.java} | 17 +-- .../MailboxElasticSearchConstants.java | 5 + .../elasticsearch/MailboxIndexerSupplier.java | 46 -------- ...sticSearchListeningMessageSearchIndex.java | 21 ++-- .../search/ElasticSearchSearcher.java | 14 +-- .../ElasticSearchIntegrationTest.java | 13 +-- ...SearchListeningMessageSearchIndexTest.java | 30 ++--- .../host/ElasticSearchHostSystem.java | 11 +- .../mailbox/ElasticSearchConfiguration.java | 80 ++++++++----- .../mailbox/ElasticSearchMailboxModule.java | 103 +++++++++-------- .../ElasticSearchConfigurationTest.java | 106 +++++++++++++++--- src/site/xdoc/server/config-elasticsearch.xml | 17 ++- 16 files changed, 281 insertions(+), 265 deletions(-) delete mode 100644 backends-common/elasticsearch/src/main/java/org/apache/james/backends/es/ElasticSearchConstants.java rename backends-common/elasticsearch/src/main/java/org/apache/james/backends/es/{Indexer.java => ElasticSearchIndexer.java} (83%) delete mode 100644 backends-common/elasticsearch/src/main/java/org/apache/james/backends/es/IndexerSupplier.java rename backends-common/elasticsearch/src/test/java/org/apache/james/backends/es/{IndexerTest.java => ElasticSearchIndexerTest.java} (94%) delete mode 100644 mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/MailboxIndexerSupplier.java diff --git a/backends-common/elasticsearch/src/main/java/org/apache/james/backends/es/DeleteByQueryPerformer.java b/backends-common/elasticsearch/src/main/java/org/apache/james/backends/es/DeleteByQueryPerformer.java index 6c8c2e3190a..a267b3a51b0 100644 --- a/backends-common/elasticsearch/src/main/java/org/apache/james/backends/es/DeleteByQueryPerformer.java +++ b/backends-common/elasticsearch/src/main/java/org/apache/james/backends/es/DeleteByQueryPerformer.java @@ -21,9 +21,6 @@ import java.util.concurrent.ExecutorService; -import javax.inject.Inject; -import javax.inject.Named; - import org.apache.james.backends.es.search.ScrollIterable; import org.elasticsearch.action.ListenableActionFuture; import org.elasticsearch.action.bulk.BulkRequestBuilder; @@ -46,13 +43,13 @@ public class DeleteByQueryPerformer { private final AliasName aliasName; private final TypeName typeName; - @Inject - public DeleteByQueryPerformer(Client client, @Named("AsyncExecutor") ExecutorService executor, @Named(ElasticSearchConstants.WRITE_ALIAS) AliasName aliasName, TypeName typeName) { + public DeleteByQueryPerformer(Client client, ExecutorService executor, + AliasName aliasName, TypeName typeName) { this(client, executor, DEFAULT_BATCH_SIZE, aliasName, typeName); } @VisibleForTesting - public DeleteByQueryPerformer(Client client, @Named("AsyncExecutor") ExecutorService executor, int batchSize, AliasName aliasName, TypeName typeName) { + public DeleteByQueryPerformer(Client client, ExecutorService executor, int batchSize, AliasName aliasName, TypeName typeName) { this.client = client; this.executor = executor; this.batchSize = batchSize; diff --git a/backends-common/elasticsearch/src/main/java/org/apache/james/backends/es/ElasticSearchConstants.java b/backends-common/elasticsearch/src/main/java/org/apache/james/backends/es/ElasticSearchConstants.java deleted file mode 100644 index 7429a3bcae5..00000000000 --- a/backends-common/elasticsearch/src/main/java/org/apache/james/backends/es/ElasticSearchConstants.java +++ /dev/null @@ -1,27 +0,0 @@ -/**************************************************************** - * Licensed to the Apache Software Foundation (ASF) under one * - * or more contributor license agreements. See the NOTICE file * - * distributed with this work for additional information * - * regarding copyright ownership. The ASF licenses this file * - * to you under the Apache License, Version 2.0 (the * - * "License"); you may not use this file except in compliance * - * with the License. You may obtain a copy of the License at * - * * - * http://www.apache.org/licenses/LICENSE-2.0 * - * * - * Unless required by applicable law or agreed to in writing, * - * software distributed under the License is distributed on an * - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * - * KIND, either express or implied. See the License for the * - * specific language governing permissions and limitations * - * under the License. * - ****************************************************************/ - -package org.apache.james.backends.es; - -public interface ElasticSearchConstants { - String WRITE_ALIAS = "injectWriteAlias"; - String READ_ALIAS = "injectReadAlias"; - - String MAILBOX_INDEX = "mailboxIndex"; -} diff --git a/backends-common/elasticsearch/src/main/java/org/apache/james/backends/es/Indexer.java b/backends-common/elasticsearch/src/main/java/org/apache/james/backends/es/ElasticSearchIndexer.java similarity index 83% rename from backends-common/elasticsearch/src/main/java/org/apache/james/backends/es/Indexer.java rename to backends-common/elasticsearch/src/main/java/org/apache/james/backends/es/ElasticSearchIndexer.java index 9a3c83b40e9..ec3f1617a2a 100644 --- a/backends-common/elasticsearch/src/main/java/org/apache/james/backends/es/Indexer.java +++ b/backends-common/elasticsearch/src/main/java/org/apache/james/backends/es/ElasticSearchIndexer.java @@ -20,6 +20,7 @@ import java.util.List; import java.util.Optional; +import java.util.concurrent.ExecutorService; import org.apache.commons.lang3.StringUtils; import org.elasticsearch.action.bulk.BulkRequestBuilder; @@ -31,23 +32,33 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; -public class Indexer { +public class ElasticSearchIndexer { private static int DEBUG_MAX_LENGTH_CONTENT = 1000; + private static final int DEFAULT_BATCH_SIZE = 100; - private static final Logger LOGGER = LoggerFactory.getLogger(Indexer.class); + private static final Logger LOGGER = LoggerFactory.getLogger(ElasticSearchIndexer.class); private final Client client; private final DeleteByQueryPerformer deleteByQueryPerformer; private final AliasName aliasName; private final TypeName typeName; - public Indexer(Client client, DeleteByQueryPerformer deleteByQueryPerformer, - AliasName aliasName, - TypeName typeName) { + public ElasticSearchIndexer(Client client, ExecutorService executor, + AliasName aliasName, + TypeName typeName) { + this(client, executor, aliasName, typeName, DEFAULT_BATCH_SIZE); + } + + @VisibleForTesting + public ElasticSearchIndexer(Client client, ExecutorService executor, + AliasName aliasName, + TypeName typeName, + int batchSize) { this.client = client; - this.deleteByQueryPerformer = deleteByQueryPerformer; + this.deleteByQueryPerformer = new DeleteByQueryPerformer(client, executor, batchSize, aliasName, typeName); this.aliasName = aliasName; this.typeName = typeName; } diff --git a/backends-common/elasticsearch/src/main/java/org/apache/james/backends/es/IndexerSupplier.java b/backends-common/elasticsearch/src/main/java/org/apache/james/backends/es/IndexerSupplier.java deleted file mode 100644 index 8a3edea2043..00000000000 --- a/backends-common/elasticsearch/src/main/java/org/apache/james/backends/es/IndexerSupplier.java +++ /dev/null @@ -1,24 +0,0 @@ -/**************************************************************** - * Licensed to the Apache Software Foundation (ASF) under one * - * or more contributor license agreements. See the NOTICE file * - * distributed with this work for additional information * - * regarding copyright ownership. The ASF licenses this file * - * to you under the Apache License, Version 2.0 (the * - * "License"); you may not use this file except in compliance * - * with the License. You may obtain a copy of the License at * - * * - * http://www.apache.org/licenses/LICENSE-2.0 * - * * - * Unless required by applicable law or agreed to in writing, * - * software distributed under the License is distributed on an * - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * - * KIND, either express or implied. See the License for the * - * specific language governing permissions and limitations * - * under the License. * - ****************************************************************/ - -package org.apache.james.backends.es; - -public interface IndexerSupplier { - Indexer getIndexer(); -} diff --git a/backends-common/elasticsearch/src/test/java/org/apache/james/backends/es/IndexerTest.java b/backends-common/elasticsearch/src/test/java/org/apache/james/backends/es/ElasticSearchIndexerTest.java similarity index 94% rename from backends-common/elasticsearch/src/test/java/org/apache/james/backends/es/IndexerTest.java rename to backends-common/elasticsearch/src/test/java/org/apache/james/backends/es/ElasticSearchIndexerTest.java index 2abbddadd01..29f6226edba 100644 --- a/backends-common/elasticsearch/src/test/java/org/apache/james/backends/es/IndexerTest.java +++ b/backends-common/elasticsearch/src/test/java/org/apache/james/backends/es/ElasticSearchIndexerTest.java @@ -27,7 +27,6 @@ import org.apache.james.backends.es.utils.TestingClientProvider; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; -import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.node.Node; import org.junit.Before; @@ -38,7 +37,7 @@ import com.google.common.collect.Lists; -public class IndexerTest { +public class ElasticSearchIndexerTest { private static final int MINIMUM_BATCH_SIZE = 1; private static final IndexName INDEX_NAME = new IndexName("index_name"); @@ -51,7 +50,7 @@ public class IndexerTest { public RuleChain ruleChain = RuleChain.outerRule(temporaryFolder).around(embeddedElasticSearch); private Node node; - private Indexer testee; + private ElasticSearchIndexer testee; @Before public void setup() { @@ -61,17 +60,7 @@ public void setup() { .useIndex(INDEX_NAME) .addAlias(ALIAS_NAME) .createIndexAndAliases(clientProvider.get()); - DeleteByQueryPerformer deleteByQueryPerformer = new DeleteByQueryPerformer(clientProvider.get(), - Executors.newSingleThreadExecutor(), - MINIMUM_BATCH_SIZE, - ALIAS_NAME, - TYPE_NAME) { - @Override - public void perform(QueryBuilder queryBuilder) { - doDeleteByQuery(queryBuilder); - } - }; - testee = new Indexer(clientProvider.get(), deleteByQueryPerformer, ALIAS_NAME, TYPE_NAME); + testee = new ElasticSearchIndexer(clientProvider.get(), Executors.newSingleThreadExecutor(), ALIAS_NAME, TYPE_NAME, MINIMUM_BATCH_SIZE); } @Test diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/MailboxElasticSearchConstants.java b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/MailboxElasticSearchConstants.java index 1017933d847..c347738058f 100644 --- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/MailboxElasticSearchConstants.java +++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/MailboxElasticSearchConstants.java @@ -24,6 +24,11 @@ import org.apache.james.backends.es.TypeName; public interface MailboxElasticSearchConstants { + + interface InjectionNames { + String MAILBOX = "mailbox"; + } + AliasName DEFAULT_MAILBOX_WRITE_ALIAS = new AliasName("mailboxWriteAlias"); AliasName DEFAULT_MAILBOX_READ_ALIAS = new AliasName("mailboxReadAlias"); IndexName DEFAULT_MAILBOX_INDEX = new IndexName("mailbox_v1"); diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/MailboxIndexerSupplier.java b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/MailboxIndexerSupplier.java deleted file mode 100644 index e8a214347f9..00000000000 --- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/MailboxIndexerSupplier.java +++ /dev/null @@ -1,46 +0,0 @@ -/**************************************************************** - * Licensed to the Apache Software Foundation (ASF) under one * - * or more contributor license agreements. See the NOTICE file * - * distributed with this work for additional information * - * regarding copyright ownership. The ASF licenses this file * - * to you under the Apache License, Version 2.0 (the * - * "License"); you may not use this file except in compliance * - * with the License. You may obtain a copy of the License at * - * * - * http://www.apache.org/licenses/LICENSE-2.0 * - * * - * Unless required by applicable law or agreed to in writing, * - * software distributed under the License is distributed on an * - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * - * KIND, either express or implied. See the License for the * - * specific language governing permissions and limitations * - * under the License. * - ****************************************************************/ -package org.apache.james.mailbox.elasticsearch; - -import javax.inject.Inject; -import javax.inject.Named; - -import org.apache.james.backends.es.AliasName; -import org.apache.james.backends.es.DeleteByQueryPerformer; -import org.apache.james.backends.es.ElasticSearchConstants; -import org.apache.james.backends.es.Indexer; -import org.apache.james.backends.es.IndexerSupplier; -import org.apache.james.backends.es.TypeName; -import org.elasticsearch.client.Client; - -public class MailboxIndexerSupplier implements IndexerSupplier { - private final Indexer indexer; - - @Inject - public MailboxIndexerSupplier(Client client, DeleteByQueryPerformer deleteByQueryPerformer, - @Named(ElasticSearchConstants.WRITE_ALIAS) AliasName aliasName, - TypeName typeName) { - this.indexer = new Indexer(client, deleteByQueryPerformer, aliasName, typeName); - } - - @Override - public Indexer getIndexer() { - return indexer; - } -} diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/events/ElasticSearchListeningMessageSearchIndex.java b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/events/ElasticSearchListeningMessageSearchIndex.java index 68d7af9ce2e..a367e597dcb 100644 --- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/events/ElasticSearchListeningMessageSearchIndex.java +++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/events/ElasticSearchListeningMessageSearchIndex.java @@ -30,14 +30,13 @@ import javax.inject.Inject; import javax.inject.Named; -import org.apache.james.backends.es.ElasticSearchConstants; -import org.apache.james.backends.es.Indexer; -import org.apache.james.backends.es.IndexerSupplier; +import org.apache.james.backends.es.ElasticSearchIndexer; import org.apache.james.backends.es.UpdatedRepresentation; import org.apache.james.mailbox.MailboxManager.MessageCapabilities; import org.apache.james.mailbox.MailboxManager.SearchCapabilities; import org.apache.james.mailbox.MailboxSession; import org.apache.james.mailbox.MessageUid; +import org.apache.james.mailbox.elasticsearch.MailboxElasticSearchConstants; import org.apache.james.mailbox.elasticsearch.json.JsonMessageConstants; import org.apache.james.mailbox.elasticsearch.json.MessageToElasticSearchJson; import org.apache.james.mailbox.elasticsearch.search.ElasticSearchSearcher; @@ -63,16 +62,16 @@ public class ElasticSearchListeningMessageSearchIndex extends ListeningMessageSe private static final Logger LOGGER = LoggerFactory.getLogger(ElasticSearchListeningMessageSearchIndex.class); private static final String ID_SEPARATOR = ":"; - private final Indexer indexer; + private final ElasticSearchIndexer elasticSearchIndexer; private final ElasticSearchSearcher searcher; private final MessageToElasticSearchJson messageToElasticSearchJson; @Inject public ElasticSearchListeningMessageSearchIndex(MessageMapperFactory factory, - @Named(ElasticSearchConstants.MAILBOX_INDEX) IndexerSupplier indexer, + @Named(MailboxElasticSearchConstants.InjectionNames.MAILBOX) ElasticSearchIndexer indexer, ElasticSearchSearcher searcher, MessageToElasticSearchJson messageToElasticSearchJson) { super(factory); - this.indexer = indexer.getIndexer(); + this.elasticSearchIndexer = indexer; this.messageToElasticSearchJson = messageToElasticSearchJson; this.searcher = searcher; } @@ -129,7 +128,7 @@ public void add(MailboxSession session, Mailbox mailbox, MailboxMessage message) mailbox.getMailboxId(), session.getUser().getUserName(), message.getUid()); - indexer.indexMessage(indexIdFor(mailbox, message.getUid()), messageToElasticSearchJson.convertToJson(message, ImmutableList.of(session.getUser()))); + elasticSearchIndexer.indexMessage(indexIdFor(mailbox, message.getUid()), messageToElasticSearchJson.convertToJson(message, ImmutableList.of(session.getUser()))); } catch (Exception e) { try { LOGGER.warn("Indexing mailbox {}-{} of user {} on message {} without attachments ", @@ -138,7 +137,7 @@ public void add(MailboxSession session, Mailbox mailbox, MailboxMessage message) session.getUser().getUserName(), message.getUid(), e); - indexer.indexMessage(indexIdFor(mailbox, message.getUid()), messageToElasticSearchJson.convertToJsonWithoutAttachment(message, ImmutableList.of(session.getUser()))); + elasticSearchIndexer.indexMessage(indexIdFor(mailbox, message.getUid()), messageToElasticSearchJson.convertToJsonWithoutAttachment(message, ImmutableList.of(session.getUser()))); } catch (JsonProcessingException e1) { LOGGER.error("Error when indexing mailbox {}-{} of user {} on message {} without its attachment", mailbox.getName(), @@ -153,7 +152,7 @@ public void add(MailboxSession session, Mailbox mailbox, MailboxMessage message) @Override public void delete(MailboxSession session, Mailbox mailbox, List expungedUids) throws MailboxException { try { - indexer.deleteMessages(expungedUids.stream() + elasticSearchIndexer.deleteMessages(expungedUids.stream() .map(uid -> indexIdFor(mailbox, uid)) .collect(Collectors.toList())); } catch (Exception e) { @@ -166,7 +165,7 @@ public void delete(MailboxSession session, Mailbox mailbox, List exp @Override public void deleteAll(MailboxSession session, Mailbox mailbox) throws MailboxException { try { - indexer.deleteAllMatchingQuery( + elasticSearchIndexer.deleteAllMatchingQuery( termQuery( JsonMessageConstants.MAILBOX_ID, mailbox.getMailboxId().serialize())); @@ -178,7 +177,7 @@ public void deleteAll(MailboxSession session, Mailbox mailbox) throws MailboxExc @Override public void update(MailboxSession session, Mailbox mailbox, List updatedFlagsList) throws MailboxException { try { - indexer.updateMessages(updatedFlagsList.stream() + elasticSearchIndexer.updateMessages(updatedFlagsList.stream() .map(updatedFlags -> createUpdatedDocumentPartFromUpdatedFlags(mailbox, updatedFlags)) .collect(Collectors.toList())); } catch (Exception e) { diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/ElasticSearchSearcher.java b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/ElasticSearchSearcher.java index cab7a53863c..68b6a32cbc1 100644 --- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/ElasticSearchSearcher.java +++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/ElasticSearchSearcher.java @@ -23,11 +23,7 @@ import java.util.Optional; import java.util.stream.Stream; -import javax.inject.Inject; -import javax.inject.Named; - import org.apache.james.backends.es.AliasName; -import org.apache.james.backends.es.ElasticSearchConstants; import org.apache.james.backends.es.TypeName; import org.apache.james.backends.es.search.ScrollIterable; import org.apache.james.mailbox.MessageUid; @@ -51,9 +47,10 @@ public class ElasticSearchSearcher { + public static final int DEFAULT_SEARCH_SIZE = 100; + private static final Logger LOGGER = LoggerFactory.getLogger(ElasticSearchSearcher.class); private static final TimeValue TIMEOUT = new TimeValue(60000); - private static final int DEFAULT_SIZE = 100; private final Client client; private final QueryConverter queryConverter; @@ -63,13 +60,6 @@ public class ElasticSearchSearcher { private final AliasName aliasName; private final TypeName typeName; - @Inject - public ElasticSearchSearcher(Client client, QueryConverter queryConverter, - MailboxId.Factory mailboxIdFactory, MessageId.Factory messageIdFactory, - @Named(ElasticSearchConstants.READ_ALIAS) AliasName aliasName, TypeName typeName) { - this(client, queryConverter, DEFAULT_SIZE, mailboxIdFactory, messageIdFactory, aliasName, typeName); - } - public ElasticSearchSearcher(Client client, QueryConverter queryConverter, int size, MailboxId.Factory mailboxIdFactory, MessageId.Factory messageIdFactory, AliasName aliasName, TypeName typeName) { diff --git a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIntegrationTest.java b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIntegrationTest.java index cc5c679ddba..444422f1116 100644 --- a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIntegrationTest.java +++ b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIntegrationTest.java @@ -25,7 +25,7 @@ import java.time.ZoneId; import java.util.concurrent.Executors; -import org.apache.james.backends.es.DeleteByQueryPerformer; +import org.apache.james.backends.es.ElasticSearchIndexer; import org.apache.james.backends.es.EmbeddedElasticSearch; import org.apache.james.backends.es.IndexCreationFactory; import org.apache.james.backends.es.NodeMappingFactory; @@ -109,14 +109,11 @@ protected void initializeMailboxManager() throws Exception { ElasticSearchListeningMessageSearchIndex elasticSearchListeningMessageSearchIndex = new ElasticSearchListeningMessageSearchIndex( storeMailboxManager.getMapperFactory(), - new MailboxIndexerSupplier(client, - new DeleteByQueryPerformer(client, - Executors.newSingleThreadExecutor(), - BATCH_SIZE, - MailboxElasticSearchConstants.DEFAULT_MAILBOX_WRITE_ALIAS, - MailboxElasticSearchConstants.MESSAGE_TYPE), + new ElasticSearchIndexer(client, + Executors.newSingleThreadExecutor(), MailboxElasticSearchConstants.DEFAULT_MAILBOX_WRITE_ALIAS, - MailboxElasticSearchConstants.MESSAGE_TYPE), + MailboxElasticSearchConstants.MESSAGE_TYPE, + BATCH_SIZE), new ElasticSearchSearcher(client, new QueryConverter(new CriterionConverter()), SEARCH_SIZE, new InMemoryId.Factory(), storeMailboxManager.getMessageIdFactory(), MailboxElasticSearchConstants.DEFAULT_MAILBOX_READ_ALIAS, diff --git a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/events/ElasticSearchListeningMessageSearchIndexTest.java b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/events/ElasticSearchListeningMessageSearchIndexTest.java index 8a078863412..0f1325ca7a4 100644 --- a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/events/ElasticSearchListeningMessageSearchIndexTest.java +++ b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/events/ElasticSearchListeningMessageSearchIndexTest.java @@ -32,7 +32,7 @@ import javax.mail.Flags; -import org.apache.james.backends.es.Indexer; +import org.apache.james.backends.es.ElasticSearchIndexer; import org.apache.james.backends.es.UpdatedRepresentation; import org.apache.james.mailbox.MailboxSession; import org.apache.james.mailbox.MailboxSession.User; @@ -67,7 +67,7 @@ public class ElasticSearchListeningMessageSearchIndexTest { public static final String EXPECTED_JSON_CONTENT = "json content"; public static final String USERNAME = "username"; - private Indexer indexer; + private ElasticSearchIndexer elasticSearchIndexer; private MessageToElasticSearchJson messageToElasticSearchJson; private ElasticSearchListeningMessageSearchIndex testee; private MailboxSession session; @@ -80,9 +80,9 @@ public void setup() throws JsonProcessingException { messageToElasticSearchJson = mock(MessageToElasticSearchJson.class); ElasticSearchSearcher elasticSearchSearcher = mock(ElasticSearchSearcher.class); - indexer = mock(Indexer.class); + elasticSearchIndexer = mock(ElasticSearchIndexer.class); - testee = new ElasticSearchListeningMessageSearchIndex(mapperFactory, indexer, elasticSearchSearcher, messageToElasticSearchJson); + testee = new ElasticSearchListeningMessageSearchIndex(mapperFactory, elasticSearchIndexer, elasticSearchSearcher, messageToElasticSearchJson); session = new MockMailboxSession(USERNAME); users = ImmutableList.of(session.getUser()); } @@ -102,7 +102,7 @@ public void addShouldIndex() throws Exception { testee.add(session, mailbox, message); //Then - verify(indexer).indexMessage(eq(ELASTIC_SEARCH_ID), eq(EXPECTED_JSON_CONTENT)); + verify(elasticSearchIndexer).indexMessage(eq(ELASTIC_SEARCH_ID), eq(EXPECTED_JSON_CONTENT)); } @SuppressWarnings("unchecked") @@ -125,7 +125,7 @@ public void addShouldIndexEmailBodyWhenNotIndexableAttachment() throws Exception testee.add(session, mailbox, message); //Then - verify(indexer).indexMessage(eq(ELASTIC_SEARCH_ID), eq(EXPECTED_JSON_CONTENT)); + verify(elasticSearchIndexer).indexMessage(eq(ELASTIC_SEARCH_ID), eq(EXPECTED_JSON_CONTENT)); } private MailboxMessage mockedMessage(MessageUid messageId) throws IOException { @@ -166,14 +166,14 @@ public void deleteShouldWork() throws Exception { .thenReturn(MAILBOX_ID); BulkResponse expectedBulkResponse = mock(BulkResponse.class); - when(indexer.deleteMessages(any(List.class))) + when(elasticSearchIndexer.deleteMessages(any(List.class))) .thenReturn(Optional.of(expectedBulkResponse)); //When testee.delete(session, mailbox, Lists.newArrayList(MESSAGE_UID)); //Then - verify(indexer).deleteMessages(eq(Lists.newArrayList(ELASTIC_SEARCH_ID))); + verify(elasticSearchIndexer).deleteMessages(eq(Lists.newArrayList(ELASTIC_SEARCH_ID))); } @Test @@ -189,14 +189,14 @@ public void deleteShouldWorkWhenMultipleMessageIds() throws Exception { .thenReturn(MAILBOX_ID); BulkResponse expectedBulkResponse = mock(BulkResponse.class); - when(indexer.deleteMessages(any(List.class))) + when(elasticSearchIndexer.deleteMessages(any(List.class))) .thenReturn(Optional.of(expectedBulkResponse)); //When testee.delete(session, mailbox, Lists.newArrayList(MESSAGE_UID, messageId2, messageId3, messageId4, messageId5)); //Then - verify(indexer).deleteMessages(eq(Lists.newArrayList(ELASTIC_SEARCH_ID, "12:2", "12:3", "12:4", "12:5"))); + verify(elasticSearchIndexer).deleteMessages(eq(Lists.newArrayList(ELASTIC_SEARCH_ID, "12:2", "12:3", "12:4", "12:5"))); } @Test @@ -207,7 +207,7 @@ public void deleteShouldNotPropagateExceptionWhenExceptionOccurs() throws Except when(mailbox.getMailboxId()) .thenReturn(MAILBOX_ID); - when(indexer.deleteMessages(any(List.class))) + when(elasticSearchIndexer.deleteMessages(any(List.class))) .thenThrow(new ElasticsearchException("")); //When @@ -241,7 +241,7 @@ public void updateShouldWork() throws Exception { //Then ImmutableList expectedUpdatedRepresentations = ImmutableList.of(new UpdatedRepresentation(ELASTIC_SEARCH_ID, "json updated content")); - verify(indexer).updateMessages(expectedUpdatedRepresentations); + verify(elasticSearchIndexer).updateMessages(expectedUpdatedRepresentations); } @Test @@ -259,7 +259,7 @@ public void updateShouldNotPropagateExceptionWhenExceptionOccurs() throws Except .thenReturn(MAILBOX_ID); ImmutableList expectedUpdatedRepresentations = ImmutableList.of(new UpdatedRepresentation(ELASTIC_SEARCH_ID, "json updated content")); - when(indexer.updateMessages(expectedUpdatedRepresentations)) + when(elasticSearchIndexer.updateMessages(expectedUpdatedRepresentations)) .thenThrow(new ElasticsearchException("")); //When @@ -281,7 +281,7 @@ public void deleteAllShouldWork() throws Exception { //Then QueryBuilder expectedQueryBuilder = QueryBuilders.termQuery("mailboxId", "12"); - verify(indexer).deleteAllMatchingQuery(refEq(expectedQueryBuilder)); + verify(elasticSearchIndexer).deleteAllMatchingQuery(refEq(expectedQueryBuilder)); } @Test @@ -292,7 +292,7 @@ public void deleteAllShouldNotPropagateExceptionWhenExceptionOccurs() throws Exc .thenReturn(MAILBOX_ID); doThrow(RuntimeException.class) - .when(indexer).deleteAllMatchingQuery(QueryBuilders.termQuery("mailboxId", "12")); + .when(elasticSearchIndexer).deleteAllMatchingQuery(QueryBuilders.termQuery("mailboxId", "12")); //When testee.deleteAll(session, mailbox); diff --git a/mpt/impl/imap-mailbox/elasticsearch/src/test/java/org/apache/james/mpt/imapmailbox/elasticsearch/host/ElasticSearchHostSystem.java b/mpt/impl/imap-mailbox/elasticsearch/src/test/java/org/apache/james/mpt/imapmailbox/elasticsearch/host/ElasticSearchHostSystem.java index 2356f61a862..cc8af21eecf 100644 --- a/mpt/impl/imap-mailbox/elasticsearch/src/test/java/org/apache/james/mpt/imapmailbox/elasticsearch/host/ElasticSearchHostSystem.java +++ b/mpt/impl/imap-mailbox/elasticsearch/src/test/java/org/apache/james/mpt/imapmailbox/elasticsearch/host/ElasticSearchHostSystem.java @@ -26,7 +26,7 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.lang.NotImplementedException; -import org.apache.james.backends.es.DeleteByQueryPerformer; +import org.apache.james.backends.es.ElasticSearchIndexer; import org.apache.james.backends.es.EmbeddedElasticSearch; import org.apache.james.backends.es.IndexCreationFactory; import org.apache.james.backends.es.NodeMappingFactory; @@ -39,7 +39,6 @@ import org.apache.james.mailbox.acl.SimpleGroupMembershipResolver; import org.apache.james.mailbox.elasticsearch.IndexAttachments; import org.apache.james.mailbox.elasticsearch.MailboxElasticSearchConstants; -import org.apache.james.mailbox.elasticsearch.MailboxIndexerSupplier; import org.apache.james.mailbox.elasticsearch.MailboxMappingFactory; import org.apache.james.mailbox.elasticsearch.events.ElasticSearchListeningMessageSearchIndex; import org.apache.james.mailbox.elasticsearch.json.MessageToElasticSearchJson; @@ -105,12 +104,14 @@ private void initFields() throws MailboxException { ElasticSearchListeningMessageSearchIndex searchIndex = new ElasticSearchListeningMessageSearchIndex( factory, - new MailboxIndexerSupplier(client, - new DeleteByQueryPerformer(client, Executors.newSingleThreadExecutor(), MailboxElasticSearchConstants.DEFAULT_MAILBOX_WRITE_ALIAS, MailboxElasticSearchConstants.MESSAGE_TYPE), + new ElasticSearchIndexer(client, + Executors.newSingleThreadExecutor(), MailboxElasticSearchConstants.DEFAULT_MAILBOX_WRITE_ALIAS, MailboxElasticSearchConstants.MESSAGE_TYPE), new ElasticSearchSearcher(client, - new QueryConverter(new CriterionConverter()), new InMemoryId.Factory(), messageIdFactory, + new QueryConverter(new CriterionConverter()), + ElasticSearchSearcher.DEFAULT_SEARCH_SIZE, + new InMemoryId.Factory(), messageIdFactory, MailboxElasticSearchConstants.DEFAULT_MAILBOX_READ_ALIAS, MailboxElasticSearchConstants.MESSAGE_TYPE), new MessageToElasticSearchJson(new DefaultTextExtractor(), ZoneId.systemDefault(), IndexAttachments.YES)); diff --git a/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/ElasticSearchConfiguration.java b/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/ElasticSearchConfiguration.java index 2fc32e992c5..f4ed4c9a718 100644 --- a/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/ElasticSearchConfiguration.java +++ b/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/ElasticSearchConfiguration.java @@ -31,6 +31,7 @@ import org.apache.james.mailbox.elasticsearch.IndexAttachments; import org.apache.james.mailbox.elasticsearch.MailboxElasticSearchConstants; import org.apache.james.util.Host; +import org.apache.james.util.OptionalUtils; import com.github.steveash.guavate.Guavate; import com.google.common.annotations.VisibleForTesting; @@ -41,10 +42,13 @@ public class ElasticSearchConfiguration { public static final String ELASTICSEARCH_MASTER_HOST = "elasticsearch.masterHost"; public static final String ELASTICSEARCH_PORT = "elasticsearch.port"; public static final String ELASTICSEARCH_INDEX_NAME = "elasticsearch.index.name"; + public static final String ELASTICSEARCH_INDEX_MAILBOX_NAME = "elasticsearch.index.mailbox.name"; public static final String ELASTICSEARCH_NB_REPLICA = "elasticsearch.nb.replica"; public static final String ELASTICSEARCH_NB_SHARDS = "elasticsearch.nb.shards"; public static final String ELASTICSEARCH_ALIAS_READ_NAME = "elasticsearch.alias.read.name"; public static final String ELASTICSEARCH_ALIAS_WRITE_NAME = "elasticsearch.alias.write.name"; + public static final String ELASTICSEARCH_ALIAS_READ_MAILBOX_NAME = "elasticsearch.alias.read.mailbox.name"; + public static final String ELASTICSEARCH_ALIAS_WRITE_MAILBOX_NAME = "elasticsearch.alias.write.mailbox.name"; public static final String ELASTICSEARCH_RETRY_CONNECTION_MIN_DELAY = "elasticsearch.retryConnection.minDelay"; public static final String ELASTICSEARCH_RETRY_CONNECTION_MAX_RETRIES = "elasticsearch.retryConnection.maxRetries"; public static final String ELASTICSEARCH_INDEX_ATTACHMENTS = "elasticsearch.indexAttachments"; @@ -76,16 +80,9 @@ public static ElasticSearchConfiguration fromProperties(PropertiesConfiguration int minDelay = configuration.getInt(ELASTICSEARCH_RETRY_CONNECTION_MIN_DELAY, DEFAULT_CONNECTION_MIN_DELAY); IndexAttachments indexAttachments = provideIndexAttachments(configuration); ImmutableList hosts = getHosts(configuration); - - AliasName readAlias = Optional.ofNullable(configuration.getString(ELASTICSEARCH_ALIAS_READ_NAME)) - .map(AliasName::new) - .orElse(MailboxElasticSearchConstants.DEFAULT_MAILBOX_READ_ALIAS); - AliasName writeAlias = Optional.ofNullable(configuration.getString(ELASTICSEARCH_ALIAS_WRITE_NAME)) - .map(AliasName::new) - .orElse(MailboxElasticSearchConstants.DEFAULT_MAILBOX_WRITE_ALIAS); - IndexName indexName = Optional.ofNullable(configuration.getString(ELASTICSEARCH_INDEX_NAME)) - .map(IndexName::new) - .orElse(MailboxElasticSearchConstants.DEFAULT_MAILBOX_INDEX); + AliasName readAlias = computeMailboxReadAlias(configuration); + AliasName writeAlias = computeMailboxWriteAlias(configuration); + IndexName indexName = computeMailboxIndexName(configuration); return new ElasticSearchConfiguration( hosts, @@ -99,6 +96,33 @@ public static ElasticSearchConfiguration fromProperties(PropertiesConfiguration indexAttachments); } + public static IndexName computeMailboxIndexName(PropertiesConfiguration configuration) { + return OptionalUtils.or( + Optional.ofNullable(configuration.getString(ELASTICSEARCH_INDEX_MAILBOX_NAME)) + .map(IndexName::new), + Optional.ofNullable(configuration.getString(ELASTICSEARCH_INDEX_NAME)) + .map(IndexName::new)) + .orElse(MailboxElasticSearchConstants.DEFAULT_MAILBOX_INDEX); + } + + public static AliasName computeMailboxWriteAlias(PropertiesConfiguration configuration) { + return OptionalUtils.or( + Optional.ofNullable(configuration.getString(ELASTICSEARCH_ALIAS_WRITE_MAILBOX_NAME)) + .map(AliasName::new), + Optional.ofNullable(configuration.getString(ELASTICSEARCH_ALIAS_WRITE_NAME)) + .map(AliasName::new)) + .orElse(MailboxElasticSearchConstants.DEFAULT_MAILBOX_WRITE_ALIAS); + } + + public static AliasName computeMailboxReadAlias(PropertiesConfiguration configuration) { + return OptionalUtils.or( + Optional.ofNullable(configuration.getString(ELASTICSEARCH_ALIAS_READ_MAILBOX_NAME)) + .map(AliasName::new), + Optional.ofNullable(configuration.getString(ELASTICSEARCH_ALIAS_READ_NAME)) + .map(AliasName::new)) + .orElse(MailboxElasticSearchConstants.DEFAULT_MAILBOX_READ_ALIAS); + } + private static IndexAttachments provideIndexAttachments(PropertiesConfiguration configuration) { if (configuration.getBoolean(ELASTICSEARCH_INDEX_ATTACHMENTS, DEFAULT_INDEX_ATTACHMENTS)) { return IndexAttachments.YES; @@ -142,22 +166,22 @@ static void validateHostsConfigurationOptions(Optional masterHost, } private final ImmutableList hosts; - private final IndexName indexName; - private final AliasName readAliasName; - private final AliasName writeAliasName; + private final IndexName indexMailboxName; + private final AliasName readAliasMailboxName; + private final AliasName writeAliasMailboxName; private final int nbShards; private final int nbReplica; private final int minDelay; private final int maxRetries; private final IndexAttachments indexAttachment; - public ElasticSearchConfiguration(ImmutableList hosts, IndexName indexName, AliasName readAliasName, - AliasName writeAliasName, int nbShards, int nbReplica, int minDelay, + public ElasticSearchConfiguration(ImmutableList hosts, IndexName indexMailboxName, AliasName readAliasMailboxName, + AliasName writeAliasMailboxName, int nbShards, int nbReplica, int minDelay, int maxRetries, IndexAttachments indexAttachment) { this.hosts = hosts; - this.indexName = indexName; - this.readAliasName = readAliasName; - this.writeAliasName = writeAliasName; + this.indexMailboxName = indexMailboxName; + this.readAliasMailboxName = readAliasMailboxName; + this.writeAliasMailboxName = writeAliasMailboxName; this.nbShards = nbShards; this.nbReplica = nbReplica; this.minDelay = minDelay; @@ -169,16 +193,16 @@ public ImmutableList getHosts() { return hosts; } - public IndexName getIndexName() { - return indexName; + public IndexName getIndexMailboxName() { + return indexMailboxName; } - public AliasName getReadAliasName() { - return readAliasName; + public AliasName getReadAliasMailboxName() { + return readAliasMailboxName; } - public AliasName getWriteAliasName() { - return writeAliasName; + public AliasName getWriteAliasMailboxName() { + return writeAliasMailboxName; } public int getNbShards() { @@ -212,16 +236,16 @@ public final boolean equals(Object o) { && Objects.equals(this.maxRetries, that.maxRetries) && Objects.equals(this.indexAttachment, that.indexAttachment) && Objects.equals(this.hosts, that.hosts) - && Objects.equals(this.indexName, that.indexName) - && Objects.equals(this.readAliasName, that.readAliasName) - && Objects.equals(this.writeAliasName, that.writeAliasName); + && Objects.equals(this.indexMailboxName, that.indexMailboxName) + && Objects.equals(this.readAliasMailboxName, that.readAliasMailboxName) + && Objects.equals(this.writeAliasMailboxName, that.writeAliasMailboxName); } return false; } @Override public final int hashCode() { - return Objects.hash(hosts, indexName, readAliasName, writeAliasName, nbShards, + return Objects.hash(hosts, indexMailboxName, readAliasMailboxName, writeAliasMailboxName, nbShards, nbReplica, minDelay, maxRetries, indexAttachment); } } diff --git a/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/ElasticSearchMailboxModule.java b/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/ElasticSearchMailboxModule.java index 811bd96e52a..a530529073f 100644 --- a/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/ElasticSearchMailboxModule.java +++ b/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/ElasticSearchMailboxModule.java @@ -19,28 +19,30 @@ package org.apache.james.modules.mailbox; +import static org.apache.james.mailbox.elasticsearch.search.ElasticSearchSearcher.DEFAULT_SEARCH_SIZE; + import java.io.FileNotFoundException; import java.time.LocalDateTime; import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; import javax.inject.Named; import javax.inject.Singleton; import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.PropertiesConfiguration; -import org.apache.james.backends.es.AliasName; import org.apache.james.backends.es.ClientProviderImpl; -import org.apache.james.backends.es.ElasticSearchConstants; +import org.apache.james.backends.es.ElasticSearchIndexer; import org.apache.james.backends.es.IndexCreationFactory; -import org.apache.james.backends.es.IndexName; -import org.apache.james.backends.es.IndexerSupplier; import org.apache.james.backends.es.NodeMappingFactory; -import org.apache.james.backends.es.TypeName; import org.apache.james.mailbox.elasticsearch.IndexAttachments; import org.apache.james.mailbox.elasticsearch.MailboxElasticSearchConstants; -import org.apache.james.mailbox.elasticsearch.MailboxIndexerSupplier; import org.apache.james.mailbox.elasticsearch.MailboxMappingFactory; import org.apache.james.mailbox.elasticsearch.events.ElasticSearchListeningMessageSearchIndex; +import org.apache.james.mailbox.elasticsearch.query.QueryConverter; +import org.apache.james.mailbox.elasticsearch.search.ElasticSearchSearcher; +import org.apache.james.mailbox.model.MailboxId; +import org.apache.james.mailbox.model.MessageId; import org.apache.james.mailbox.store.search.ListeningMessageSearchIndex; import org.apache.james.mailbox.store.search.MessageSearchIndex; import org.apache.james.utils.PropertiesProvider; @@ -53,7 +55,6 @@ import com.google.inject.AbstractModule; import com.google.inject.Provides; import com.google.inject.Scopes; -import com.google.inject.name.Names; import com.nurkiewicz.asyncretry.AsyncRetryExecutor; public class ElasticSearchMailboxModule extends AbstractModule { @@ -63,14 +64,39 @@ public class ElasticSearchMailboxModule extends AbstractModule { @Override protected void configure() { - bind(TypeName.class).toInstance(MailboxElasticSearchConstants.MESSAGE_TYPE); bind(ElasticSearchListeningMessageSearchIndex.class).in(Scopes.SINGLETON); bind(MessageSearchIndex.class).to(ElasticSearchListeningMessageSearchIndex.class); bind(ListeningMessageSearchIndex.class).to(ElasticSearchListeningMessageSearchIndex.class); - bind(MailboxIndexerSupplier.class).in(Scopes.SINGLETON); - bind(IndexerSupplier.class) - .annotatedWith(Names.named(ElasticSearchConstants.MAILBOX_INDEX)) - .to(MailboxIndexerSupplier.class); + } + + @Provides + @Singleton + @Named(MailboxElasticSearchConstants.InjectionNames.MAILBOX) + private ElasticSearchIndexer createMailboxElasticSearchIndexer(Client client, + @Named("AsyncExecutor") ExecutorService executor, + ElasticSearchConfiguration configuration) { + return new ElasticSearchIndexer( + client, + executor, + configuration.getWriteAliasMailboxName(), + MailboxElasticSearchConstants.MESSAGE_TYPE); + } + + @Provides + @Singleton + private ElasticSearchSearcher createMailboxElasticSearchSearcher(Client client, + QueryConverter queryConverter, + MailboxId.Factory mailboxIdFactory, + MessageId.Factory messageIdFactory, + ElasticSearchConfiguration configuration) { + return new ElasticSearchSearcher( + client, + queryConverter, + DEFAULT_SEARCH_SIZE, + mailboxIdFactory, + messageIdFactory, + configuration.getReadAliasMailboxName(), + MailboxElasticSearchConstants.MESSAGE_TYPE); } @Provides @@ -85,55 +111,38 @@ private ElasticSearchConfiguration getElasticSearchConfiguration(PropertiesProvi } } - @Provides - protected IndexName provideIndexName(ElasticSearchConfiguration configuration) { - return configuration.getIndexName(); - } - - @Provides - @Named(ElasticSearchConstants.READ_ALIAS) - protected AliasName provideReadAliasName(ElasticSearchConfiguration configuration) { - return configuration.getReadAliasName(); - } - - @Provides - @Named(ElasticSearchConstants.WRITE_ALIAS) - protected AliasName provideWriteAliasName(ElasticSearchConfiguration configuration) { - return configuration.getWriteAliasName(); - } - - @Provides - @Singleton - protected IndexCreationFactory provideIndexCreationFactory(ElasticSearchConfiguration configuration) { - return new IndexCreationFactory() - .useIndex(configuration.getIndexName()) - .addAlias(configuration.getReadAliasName()) - .addAlias(configuration.getWriteAliasName()) - .nbShards(configuration.getNbShards()) - .nbReplica(configuration.getNbReplica()); - } - @Provides @Singleton protected Client provideClient(ElasticSearchConfiguration configuration, - IndexCreationFactory indexCreationFactory, - AsyncRetryExecutor executor) throws ExecutionException, InterruptedException { + AsyncRetryExecutor executor) throws ExecutionException, InterruptedException { return RetryExecutorUtil.retryOnExceptions(executor, configuration.getMaxRetries(), configuration.getMinDelay(), NoNodeAvailableException.class) - .getWithRetry(context -> connectToCluster(configuration, indexCreationFactory)) + .getWithRetry(context -> connectToCluster(configuration)) .get(); } - private Client connectToCluster(ElasticSearchConfiguration configuration, IndexCreationFactory indexCreationFactory) { + private Client connectToCluster(ElasticSearchConfiguration configuration) { LOGGER.info("Trying to connect to ElasticSearch service at {}", LocalDateTime.now()); Client client = ClientProviderImpl.fromHosts(configuration.getHosts()).get(); - indexCreationFactory.createIndexAndAliases(client); - return NodeMappingFactory.applyMapping(client, - configuration.getIndexName(), + createMailboxIndexCreationFactory(configuration).createIndexAndAliases(client); + + NodeMappingFactory.applyMapping(client, + configuration.getIndexMailboxName(), MailboxElasticSearchConstants.MESSAGE_TYPE, MailboxMappingFactory.getMappingContent()); + + return client; + } + + protected IndexCreationFactory createMailboxIndexCreationFactory(ElasticSearchConfiguration configuration) { + return new IndexCreationFactory() + .useIndex(configuration.getIndexMailboxName()) + .addAlias(configuration.getReadAliasMailboxName()) + .addAlias(configuration.getWriteAliasMailboxName()) + .nbShards(configuration.getNbShards()) + .nbReplica(configuration.getNbReplica()); } @Provides diff --git a/server/container/guice/cassandra-guice/src/test/java/org/apache/james/modules/mailbox/ElasticSearchConfigurationTest.java b/server/container/guice/cassandra-guice/src/test/java/org/apache/james/modules/mailbox/ElasticSearchConfigurationTest.java index 9c1a3fc2e48..e6b13d33001 100644 --- a/server/container/guice/cassandra-guice/src/test/java/org/apache/james/modules/mailbox/ElasticSearchConfigurationTest.java +++ b/server/container/guice/cassandra-guice/src/test/java/org/apache/james/modules/mailbox/ElasticSearchConfigurationTest.java @@ -134,7 +134,7 @@ public void getMinDelayShouldReturnDefaultValueWhenMissing() throws Configuratio } @Test - public void getIndexNameShouldReturnConfiguredValue() throws ConfigurationException { + public void getIndexMailboxNameShouldReturnOldConfiguredValue() throws ConfigurationException { PropertiesConfiguration configuration = new PropertiesConfiguration(); String name = "name"; configuration.addProperty("elasticsearch.index.name", name); @@ -142,23 +142,50 @@ public void getIndexNameShouldReturnConfiguredValue() throws ConfigurationExcept ElasticSearchConfiguration elasticSearchConfiguration = ElasticSearchConfiguration.fromProperties(configuration); - assertThat(elasticSearchConfiguration.getIndexName()) + assertThat(elasticSearchConfiguration.getIndexMailboxName()) .isEqualTo(new IndexName(name)); } @Test - public void getIndexNameShouldReturnDefaultValueWhenMissing() throws ConfigurationException { + public void getIndexMailboxNameShouldReturnNewConfiguredValueWhenBoth() throws ConfigurationException { PropertiesConfiguration configuration = new PropertiesConfiguration(); + String name = "name"; + configuration.addProperty("elasticsearch.index.name", "other"); + configuration.addProperty("elasticsearch.index.mailbox.name", name); + configuration.addProperty("elasticsearch.hosts", "127.0.0.1"); + + ElasticSearchConfiguration elasticSearchConfiguration = ElasticSearchConfiguration.fromProperties(configuration); + + assertThat(elasticSearchConfiguration.getIndexMailboxName()) + .isEqualTo(new IndexName(name)); + } + + @Test + public void getIndexMailboxNameShouldReturnConfiguredValue() throws ConfigurationException { + PropertiesConfiguration configuration = new PropertiesConfiguration(); + String name = "name"; + configuration.addProperty("elasticsearch.index.mailbox.name", name); configuration.addProperty("elasticsearch.hosts", "127.0.0.1"); ElasticSearchConfiguration elasticSearchConfiguration = ElasticSearchConfiguration.fromProperties(configuration); - assertThat(elasticSearchConfiguration.getIndexName()) + assertThat(elasticSearchConfiguration.getIndexMailboxName()) + .isEqualTo(new IndexName(name)); + } + + @Test + public void getIndexMailboxNameShouldReturnDefaultValueWhenMissing() throws ConfigurationException { + PropertiesConfiguration configuration = new PropertiesConfiguration(); + configuration.addProperty("elasticsearch.hosts", "127.0.0.1"); + + ElasticSearchConfiguration elasticSearchConfiguration = ElasticSearchConfiguration.fromProperties(configuration); + + assertThat(elasticSearchConfiguration.getIndexMailboxName()) .isEqualTo(MailboxElasticSearchConstants.DEFAULT_MAILBOX_INDEX); } @Test - public void getReadAliasNameShouldReturnConfiguredValue() throws ConfigurationException { + public void getReadAliasMailboxNameShouldReturnOldConfiguredValue() throws ConfigurationException { PropertiesConfiguration configuration = new PropertiesConfiguration(); String name = "name"; configuration.addProperty("elasticsearch.alias.read.name", name); @@ -166,24 +193,50 @@ public void getReadAliasNameShouldReturnConfiguredValue() throws ConfigurationEx ElasticSearchConfiguration elasticSearchConfiguration = ElasticSearchConfiguration.fromProperties(configuration); - assertThat(elasticSearchConfiguration.getReadAliasName()) + assertThat(elasticSearchConfiguration.getReadAliasMailboxName()) .isEqualTo(new AliasName(name)); } @Test - public void getReadAliasNameShouldReturnDefaultValueWhenMissing() throws ConfigurationException { + public void getReadAliasMailboxNameShouldReturnConfiguredValue() throws ConfigurationException { PropertiesConfiguration configuration = new PropertiesConfiguration(); + String name = "name"; + configuration.addProperty("elasticsearch.alias.read.mailbox.name", name); configuration.addProperty("elasticsearch.hosts", "127.0.0.1"); ElasticSearchConfiguration elasticSearchConfiguration = ElasticSearchConfiguration.fromProperties(configuration); - assertThat(elasticSearchConfiguration.getReadAliasName()) - .isEqualTo(MailboxElasticSearchConstants.DEFAULT_MAILBOX_READ_ALIAS); + assertThat(elasticSearchConfiguration.getReadAliasMailboxName()) + .isEqualTo(new AliasName(name)); } + @Test + public void getReadAliasMailboxNameShouldReturnNewConfiguredValueWhenBoth() throws ConfigurationException { + PropertiesConfiguration configuration = new PropertiesConfiguration(); + String name = "name"; + configuration.addProperty("elasticsearch.alias.read.mailbox.name", name); + configuration.addProperty("elasticsearch.alias.read.name", "other"); + configuration.addProperty("elasticsearch.hosts", "127.0.0.1"); + + ElasticSearchConfiguration elasticSearchConfiguration = ElasticSearchConfiguration.fromProperties(configuration); + + assertThat(elasticSearchConfiguration.getReadAliasMailboxName()) + .isEqualTo(new AliasName(name)); + } @Test - public void getWriteAliasNameNameShouldReturnConfiguredValue() throws ConfigurationException { + public void getReadAliasMailboxNameShouldReturnDefaultValueWhenMissing() throws ConfigurationException { + PropertiesConfiguration configuration = new PropertiesConfiguration(); + configuration.addProperty("elasticsearch.hosts", "127.0.0.1"); + + ElasticSearchConfiguration elasticSearchConfiguration = ElasticSearchConfiguration.fromProperties(configuration); + + assertThat(elasticSearchConfiguration.getReadAliasMailboxName()) + .isEqualTo(MailboxElasticSearchConstants.DEFAULT_MAILBOX_READ_ALIAS); + } + + @Test + public void getWriteAliasMailboxNameShouldReturnOldConfiguredValue() throws ConfigurationException { PropertiesConfiguration configuration = new PropertiesConfiguration(); String name = "name"; configuration.addProperty("elasticsearch.alias.write.name", name); @@ -191,18 +244,45 @@ public void getWriteAliasNameNameShouldReturnConfiguredValue() throws Configurat ElasticSearchConfiguration elasticSearchConfiguration = ElasticSearchConfiguration.fromProperties(configuration); - assertThat(elasticSearchConfiguration.getWriteAliasName()) + assertThat(elasticSearchConfiguration.getWriteAliasMailboxName()) + .isEqualTo(new AliasName(name)); + } + + @Test + public void getWriteAliasMailboxNameShouldReturnConfiguredValue() throws ConfigurationException { + PropertiesConfiguration configuration = new PropertiesConfiguration(); + String name = "name"; + configuration.addProperty("elasticsearch.alias.write.mailbox.name", name); + configuration.addProperty("elasticsearch.hosts", "127.0.0.1"); + + ElasticSearchConfiguration elasticSearchConfiguration = ElasticSearchConfiguration.fromProperties(configuration); + + assertThat(elasticSearchConfiguration.getWriteAliasMailboxName()) + .isEqualTo(new AliasName(name)); + } + + @Test + public void getWriteAliasMailboxNameShouldReturnNewConfiguredValueWhenBoth() throws ConfigurationException { + PropertiesConfiguration configuration = new PropertiesConfiguration(); + String name = "name"; + configuration.addProperty("elasticsearch.alias.write.mailbox.name", name); + configuration.addProperty("elasticsearch.alias.write.name", "other"); + configuration.addProperty("elasticsearch.hosts", "127.0.0.1"); + + ElasticSearchConfiguration elasticSearchConfiguration = ElasticSearchConfiguration.fromProperties(configuration); + + assertThat(elasticSearchConfiguration.getWriteAliasMailboxName()) .isEqualTo(new AliasName(name)); } @Test - public void getWriteAliasNameShouldReturnDefaultValueWhenMissing() throws ConfigurationException { + public void getWriteAliasMailboxNameShouldReturnDefaultValueWhenMissing() throws ConfigurationException { PropertiesConfiguration configuration = new PropertiesConfiguration(); configuration.addProperty("elasticsearch.hosts", "127.0.0.1"); ElasticSearchConfiguration elasticSearchConfiguration = ElasticSearchConfiguration.fromProperties(configuration); - assertThat(elasticSearchConfiguration.getWriteAliasName()) + assertThat(elasticSearchConfiguration.getWriteAliasMailboxName()) .isEqualTo(MailboxElasticSearchConstants.DEFAULT_MAILBOX_WRITE_ALIAS); } diff --git a/src/site/xdoc/server/config-elasticsearch.xml b/src/site/xdoc/server/config-elasticsearch.xml index b5ca6c80793..d1fb1818b9b 100644 --- a/src/site/xdoc/server/config-elasticsearch.xml +++ b/src/site/xdoc/server/config-elasticsearch.xml @@ -56,13 +56,24 @@
Number of shards for index provisionned by James
elasticsearch.nb.replica
Number of replica for index provisionned by James
+
elasticsearch.index.mailbox.name
+
Name of the mailbox index backed by the alias. It will be created if missing.
elasticsearch.index.name
-
Name of the index backed by the alias. It will be created if missing.
+
Deprecated Use elasticsearch.index.mailbox.name instead.
+ Name of the mailbox index backed by the alias. It will be created if missing.
+
elasticsearch.alias.read.mailbox.name
+
Name of the alias to use by Apache James for mailbox reads. It will be created if missing. + The target of the alias is the index name configured above.
elasticsearch.alias.read.name
-
Name of the alias to use by Apache James for reads. It will be created if missing. +
Deprecated Use elasticsearch.alias.read.mailbox.name instead.
+ Name of the alias to use by Apache James for mailbox reads. It will be created if missing. + The target of the alias is the index name configured above.
+
elasticsearch.alias.write.mailbox.name
+
Name of the alias to use by Apache James for mailbox writes. It will be created if missing. The target of the alias is the index name configured above.
elasticsearch.alias.write.name
-
Name of the alias to use by Apache James for writes. It will be created if missing. +
Deprecated Use elasticsearch.alias.write.mailbox.name instead.
+ Name of the alias to use by Apache James for mailbox writes. It will be created if missing. The target of the alias is the index name configured above.
elasticsearch.retryConnection.maxRetries
Number of retries when connecting the cluster