From 13818d6694ff7e6007c25c36c1259da095268824 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20G=C3=A9raud?= Date: Tue, 30 Apr 2019 15:57:44 +0200 Subject: [PATCH] feat(search): synchronize search indexes across management instances fix gravitee-io/issues#2166 --- .../model/command/CommandEntity.java | 71 ++++++++ .../model/command/CommandQuery.java | 44 +++++ .../command/CommandSearchIndexerEntity.java | 60 +++++++ .../management/model/command/CommandTags.java | 25 +++ .../model/command/NewCommandEntity.java | 62 +++++++ .../management/model/search/Indexable.java | 2 + .../proxy/CommandRepositoryProxy.java | 56 +++++++ gravitee-management-api-service/pom.xml | 6 +- .../management/service/CommandService.java | 33 ++++ .../Message2RecipientNotFoundException.java | 36 ++++ .../service/impl/ApiServiceImpl.java | 4 +- .../service/impl/CommandServiceImpl.java | 156 ++++++++++++++++++ .../service/impl/MessageServiceImpl.java | 4 +- .../service/impl/PageServiceImpl.java | 6 +- .../impl/search/SearchEngineServiceImpl.java | 105 +++++++++++- .../impl/upgrade/SearchIndexUpgrader.java | 4 +- .../service/search/SearchEngineService.java | 7 +- .../pom.xml | 85 ++++++++++ .../src/main/assembly/plugin-assembly.xml | 40 +++++ .../search/ScheduledSearchIndexerService.java | 105 ++++++++++++ .../spring/SearchIndexerConfiguration.java | 36 ++++ .../src/main/resources/plugin.properties | 22 +++ .../ScheduledSearchIndexerServiceTest.java | 77 +++++++++ gravitee-management-api-services/pom.xml | 1 + pom.xml | 2 +- 25 files changed, 1028 insertions(+), 21 deletions(-) create mode 100644 gravitee-management-api-model/src/main/java/io/gravitee/management/model/command/CommandEntity.java create mode 100644 gravitee-management-api-model/src/main/java/io/gravitee/management/model/command/CommandQuery.java create mode 100644 gravitee-management-api-model/src/main/java/io/gravitee/management/model/command/CommandSearchIndexerEntity.java create mode 100644 gravitee-management-api-model/src/main/java/io/gravitee/management/model/command/CommandTags.java create mode 100644 gravitee-management-api-model/src/main/java/io/gravitee/management/model/command/NewCommandEntity.java create mode 100644 gravitee-management-api-repository/src/main/java/io/gravitee/management/repository/proxy/CommandRepositoryProxy.java create mode 100644 gravitee-management-api-service/src/main/java/io/gravitee/management/service/CommandService.java create mode 100644 gravitee-management-api-service/src/main/java/io/gravitee/management/service/exceptions/Message2RecipientNotFoundException.java create mode 100644 gravitee-management-api-service/src/main/java/io/gravitee/management/service/impl/CommandServiceImpl.java create mode 100644 gravitee-management-api-services/gravitee-management-api-services-search-indexer/pom.xml create mode 100644 gravitee-management-api-services/gravitee-management-api-services-search-indexer/src/main/assembly/plugin-assembly.xml create mode 100644 gravitee-management-api-services/gravitee-management-api-services-search-indexer/src/main/java/io/gravitee/management/services/search/ScheduledSearchIndexerService.java create mode 100644 gravitee-management-api-services/gravitee-management-api-services-search-indexer/src/main/java/io/gravitee/management/services/search/spring/SearchIndexerConfiguration.java create mode 100644 gravitee-management-api-services/gravitee-management-api-services-search-indexer/src/main/resources/plugin.properties create mode 100644 gravitee-management-api-services/gravitee-management-api-services-search-indexer/src/test/java/io/gravitee/management/services/search/ScheduledSearchIndexerServiceTest.java diff --git a/gravitee-management-api-model/src/main/java/io/gravitee/management/model/command/CommandEntity.java b/gravitee-management-api-model/src/main/java/io/gravitee/management/model/command/CommandEntity.java new file mode 100644 index 0000000000..90497620a1 --- /dev/null +++ b/gravitee-management-api-model/src/main/java/io/gravitee/management/model/command/CommandEntity.java @@ -0,0 +1,71 @@ +/** + * Copyright (C) 2015 The Gravitee team (http://gravitee.io) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.gravitee.management.model.command; + +import java.util.List; + +/** + * @author Nicolas GERAUD (nicolas.geraud at graviteesource.com) + * @author GraviteeSource Team + */ +public class CommandEntity { + + private String id; + private String to; + private List tags; + private String content; + private long ttlInSeconds; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getTo() { + return to; + } + + public void setTo(String to) { + this.to = to; + } + + public List getTags() { + return tags; + } + + public void setTags(List tags) { + this.tags = tags; + } + + public String getContent() { + return content; + } + + public void setContent(String content) { + this.content = content; + } + + public long getTtlInSeconds() { + return ttlInSeconds; + } + + public void setTtlInSeconds(long ttlInSeconds) { + this.ttlInSeconds = ttlInSeconds; + } +} diff --git a/gravitee-management-api-model/src/main/java/io/gravitee/management/model/command/CommandQuery.java b/gravitee-management-api-model/src/main/java/io/gravitee/management/model/command/CommandQuery.java new file mode 100644 index 0000000000..45a1544758 --- /dev/null +++ b/gravitee-management-api-model/src/main/java/io/gravitee/management/model/command/CommandQuery.java @@ -0,0 +1,44 @@ +/** + * Copyright (C) 2015 The Gravitee team (http://gravitee.io) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.gravitee.management.model.command; + +import java.util.List; + +/** + * @author Nicolas GERAUD (nicolas.geraud at graviteesource.com) + * @author GraviteeSource Team + */ +public class CommandQuery { + + private String to; + private List tags; + + public String getTo() { + return to; + } + + public void setTo(String to) { + this.to = to; + } + + public List getTags() { + return tags; + } + + public void setTags(List tags) { + this.tags = tags; + } +} diff --git a/gravitee-management-api-model/src/main/java/io/gravitee/management/model/command/CommandSearchIndexerEntity.java b/gravitee-management-api-model/src/main/java/io/gravitee/management/model/command/CommandSearchIndexerEntity.java new file mode 100644 index 0000000000..36ebd1d9c1 --- /dev/null +++ b/gravitee-management-api-model/src/main/java/io/gravitee/management/model/command/CommandSearchIndexerEntity.java @@ -0,0 +1,60 @@ +/** + * Copyright (C) 2015 The Gravitee team (http://gravitee.io) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.gravitee.management.model.command; + +/** + * @author Nicolas GERAUD (nicolas.geraud at graviteesource.com) + * @author GraviteeSource Team + */ +public class CommandSearchIndexerEntity { + + private String id; + private String clazz; + private String action; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getClazz() { + return clazz; + } + + public void setClazz(String clazz) { + this.clazz = clazz; + } + + public String getAction() { + return action; + } + + public void setAction(String action) { + this.action = action; + } + + @Override + public String toString() { + return "CommandSearchIndexerEntity{" + + "id='" + id + '\'' + + ", clazz='" + clazz + '\'' + + ", action='" + action + '\'' + + '}'; + } +} diff --git a/gravitee-management-api-model/src/main/java/io/gravitee/management/model/command/CommandTags.java b/gravitee-management-api-model/src/main/java/io/gravitee/management/model/command/CommandTags.java new file mode 100644 index 0000000000..4f263f1d3c --- /dev/null +++ b/gravitee-management-api-model/src/main/java/io/gravitee/management/model/command/CommandTags.java @@ -0,0 +1,25 @@ +/** + * Copyright (C) 2015 The Gravitee team (http://gravitee.io) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.gravitee.management.model.command; + +/** + * @author Nicolas GERAUD (nicolas.geraud at graviteesource.com) + * @author GraviteeSource Team + */ + +public enum CommandTags { + DATA_TO_INDEX +} diff --git a/gravitee-management-api-model/src/main/java/io/gravitee/management/model/command/NewCommandEntity.java b/gravitee-management-api-model/src/main/java/io/gravitee/management/model/command/NewCommandEntity.java new file mode 100644 index 0000000000..e52678555c --- /dev/null +++ b/gravitee-management-api-model/src/main/java/io/gravitee/management/model/command/NewCommandEntity.java @@ -0,0 +1,62 @@ +/** + * Copyright (C) 2015 The Gravitee team (http://gravitee.io) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.gravitee.management.model.command; + +import java.util.List; + +/** + * @author Nicolas GERAUD (nicolas.geraud at graviteesource.com) + * @author GraviteeSource Team + */ +public class NewCommandEntity { + + private String to; + private List tags; + private String content; + private long ttlInSeconds; + + public String getTo() { + return to; + } + + public void setTo(String to) { + this.to = to; + } + + public List getTags() { + return tags; + } + + public void setTags(List tags) { + this.tags = tags; + } + + public String getContent() { + return content; + } + + public void setContent(String content) { + this.content = content; + } + + public long getTtlInSeconds() { + return ttlInSeconds; + } + + public void setTtlInSeconds(long ttlInSeconds) { + this.ttlInSeconds = ttlInSeconds; + } +} diff --git a/gravitee-management-api-model/src/main/java/io/gravitee/management/model/search/Indexable.java b/gravitee-management-api-model/src/main/java/io/gravitee/management/model/search/Indexable.java index df4b9fa1ff..33ed88e402 100644 --- a/gravitee-management-api-model/src/main/java/io/gravitee/management/model/search/Indexable.java +++ b/gravitee-management-api-model/src/main/java/io/gravitee/management/model/search/Indexable.java @@ -17,7 +17,9 @@ /** * @author David BRASSELY (david.brassely at graviteesource.com) + * @author Nicolas GERAUD (nicolas.geraud at graviteesource.com) * @author GraviteeSource Team */ public interface Indexable { + String getId(); } diff --git a/gravitee-management-api-repository/src/main/java/io/gravitee/management/repository/proxy/CommandRepositoryProxy.java b/gravitee-management-api-repository/src/main/java/io/gravitee/management/repository/proxy/CommandRepositoryProxy.java new file mode 100644 index 0000000000..6e7020e224 --- /dev/null +++ b/gravitee-management-api-repository/src/main/java/io/gravitee/management/repository/proxy/CommandRepositoryProxy.java @@ -0,0 +1,56 @@ +/** + * Copyright (C) 2015 The Gravitee team (http://gravitee.io) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.gravitee.management.repository.proxy; + +import io.gravitee.repository.exceptions.TechnicalException; +import io.gravitee.repository.management.api.CommandRepository; +import io.gravitee.repository.management.api.search.CommandCriteria; +import io.gravitee.repository.management.model.Command; +import org.springframework.stereotype.Component; + +import java.util.List; +import java.util.Optional; + +/** + * @author Nicolas GERAUD (nicolas.geraud at graviteesource.com) + * @author GraviteeSource Team + */ +@Component +public class CommandRepositoryProxy extends AbstractProxy implements CommandRepository { + @Override + public Optional findById(String id) throws TechnicalException { + return target.findById(id); + } + + @Override + public Command create(Command command) throws TechnicalException { + return target.create(command); + } + + @Override + public Command update(Command command) throws TechnicalException { + return target.update(command); } + + @Override + public void delete(String id) throws TechnicalException { + target.delete(id); + } + + @Override + public List search(CommandCriteria criteria) { + return target.search(criteria); + } +} diff --git a/gravitee-management-api-service/pom.xml b/gravitee-management-api-service/pom.xml index 40403b6927..e274373005 100644 --- a/gravitee-management-api-service/pom.xml +++ b/gravitee-management-api-service/pom.xml @@ -228,5 +228,9 @@ ${powermock.version} test - + + io.gravitee.node + gravitee-node-api + + diff --git a/gravitee-management-api-service/src/main/java/io/gravitee/management/service/CommandService.java b/gravitee-management-api-service/src/main/java/io/gravitee/management/service/CommandService.java new file mode 100644 index 0000000000..6d21c8e2fe --- /dev/null +++ b/gravitee-management-api-service/src/main/java/io/gravitee/management/service/CommandService.java @@ -0,0 +1,33 @@ +/** + * Copyright (C) 2015 The Gravitee team (http://gravitee.io) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.gravitee.management.service; + +import io.gravitee.management.model.command.CommandEntity; +import io.gravitee.management.model.command.CommandQuery; +import io.gravitee.management.model.command.NewCommandEntity; + +import java.util.List; + +/** + * @author Nicolas GERAUD (nicolas.geraud at graviteesource.com) + * @author GraviteeSource Team + */ +public interface CommandService { + + void send(NewCommandEntity message); + List search(CommandQuery query); + void ack(String messageId); +} diff --git a/gravitee-management-api-service/src/main/java/io/gravitee/management/service/exceptions/Message2RecipientNotFoundException.java b/gravitee-management-api-service/src/main/java/io/gravitee/management/service/exceptions/Message2RecipientNotFoundException.java new file mode 100644 index 0000000000..d281985e58 --- /dev/null +++ b/gravitee-management-api-service/src/main/java/io/gravitee/management/service/exceptions/Message2RecipientNotFoundException.java @@ -0,0 +1,36 @@ +/** + * Copyright (C) 2015 The Gravitee team (http://gravitee.io) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.gravitee.management.service.exceptions; + +import io.gravitee.common.http.HttpStatusCode; + +/** + * @author Nicolas GERAUD (nicolas.geraud at graviteesource.com) + * @author GraviteeSource Team + */ +public class Message2RecipientNotFoundException extends AbstractManagementException { + + + @Override + public int getHttpStatusCode() { + return HttpStatusCode.BAD_REQUEST_400; + } + + @Override + public String getMessage() { + return "Message recipients are missing."; + } +} diff --git a/gravitee-management-api-service/src/main/java/io/gravitee/management/service/impl/ApiServiceImpl.java b/gravitee-management-api-service/src/main/java/io/gravitee/management/service/impl/ApiServiceImpl.java index 47f8368592..bc6e8ab48d 100644 --- a/gravitee-management-api-service/src/main/java/io/gravitee/management/service/impl/ApiServiceImpl.java +++ b/gravitee-management-api-service/src/main/java/io/gravitee/management/service/impl/ApiServiceImpl.java @@ -272,7 +272,7 @@ private ApiEntity create0(UpdateApiEntity api, String userId) throws ApiAlreadyE //TODO add membership log ApiEntity apiEntity = convert(createdApi, primaryOwner); - searchEngineService.index(apiEntity); + searchEngineService.index(apiEntity, false); return apiEntity; } else { LOGGER.error("Unable to create API {} because of previous error.", api.getName()); @@ -535,7 +535,7 @@ public ApiEntity update(String apiId, UpdateApiEntity updateApiEntity) { updatedApi); ApiEntity apiEntity = convert(singletonList(updatedApi)).iterator().next(); - searchEngineService.index(apiEntity); + searchEngineService.index(apiEntity, false); return apiEntity; } else { LOGGER.error("Unable to update API {} because of previous error.", api.getId()); diff --git a/gravitee-management-api-service/src/main/java/io/gravitee/management/service/impl/CommandServiceImpl.java b/gravitee-management-api-service/src/main/java/io/gravitee/management/service/impl/CommandServiceImpl.java new file mode 100644 index 0000000000..cdaf94d406 --- /dev/null +++ b/gravitee-management-api-service/src/main/java/io/gravitee/management/service/impl/CommandServiceImpl.java @@ -0,0 +1,156 @@ +/** + * Copyright (C) 2015 The Gravitee team (http://gravitee.io) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.gravitee.management.service.impl; + +import io.gravitee.common.utils.UUID; +import io.gravitee.management.model.command.CommandEntity; +import io.gravitee.management.model.command.CommandQuery; +import io.gravitee.management.model.command.CommandTags; +import io.gravitee.management.model.command.NewCommandEntity; +import io.gravitee.management.service.CommandService; +import io.gravitee.management.service.exceptions.Message2RecipientNotFoundException; +import io.gravitee.management.service.exceptions.TechnicalManagementException; +import io.gravitee.node.api.Node; +import io.gravitee.repository.exceptions.TechnicalException; +import io.gravitee.repository.management.api.CommandRepository; +import io.gravitee.repository.management.api.search.CommandCriteria; +import io.gravitee.repository.management.model.Command; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * @author Nicolas GERAUD (nicolas.geraud at graviteesource.com) + * @author GraviteeSource Team + */ +@Component +public class CommandServiceImpl extends AbstractService implements CommandService { + + private final Logger logger = LoggerFactory.getLogger(CommandServiceImpl.class); + + @Autowired + CommandRepository commandRepository; + + @Autowired + Node node; + + @Override + public void send(NewCommandEntity messageEntity) { + if (messageEntity.getTo() == null || messageEntity.getTo().isEmpty()) { + throw new Message2RecipientNotFoundException(); + } + + Command command = new Command(); + command.setId(UUID.toString(java.util.UUID.randomUUID())); + command.setFrom(node.id()); + command.setTo(messageEntity.getTo()); + command.setTags(convert(messageEntity.getTags())); + long now = System.currentTimeMillis(); + command.setCreatedAt(new Date(now)); + command.setUpdatedAt(command.getCreatedAt()); + command.setExpiredAt(new Date(now + (messageEntity.getTtlInSeconds() * 1000))); + if (messageEntity.getContent() != null) { + command.setContent(messageEntity.getContent()); + } + + try { + commandRepository.create(command); + } catch (TechnicalException ex) { + logger.error("An error occurs while trying to create {}", command, ex); + throw new TechnicalManagementException("An error occurs while trying create " + command, ex); + } + } + + @Override + public List search(CommandQuery query) { + //convert tags + String[] tags = null; + if (query.getTags() != null) { + tags = query.getTags() + .stream() + .map(Enum::name) + .toArray(String[]::new); + } + CommandCriteria criteria = new CommandCriteria.Builder() + .to(query.getTo()) + .tags(tags) + .notAckBy(node.id()) + .notDeleted() + .build(); + return commandRepository.search(criteria) + .stream() + .map(this::map) + .collect(Collectors.toList()); + } + + @Override + public void ack(String messageId) { + try { + Optional optMsg = commandRepository.findById(messageId); + //if not found, this is probably because it has been deleted + if (optMsg.isPresent()) { + Command msg = optMsg.get(); + if (msg.getAcknowledgments() == null) { + msg.setAcknowledgments(Collections.singletonList(node.id())); + } else { + msg.getAcknowledgments().add(node.id()); + } + commandRepository.update(msg); + } + } catch (TechnicalException ex) { + logger.error("An error occurs while trying to acknowledge a message", ex); + } + } + + private List convert(List tags) { + if (tags == null || tags.isEmpty()) { + return Collections.emptyList(); + } + + return tags + .stream() + .map(Enum::name) + .collect(Collectors.toList()); + } + + private CommandEntity map(Command command) { + if (command == null) { + return null; + } + + CommandEntity commandEntity = new CommandEntity(); + + commandEntity.setId(command.getId()); + commandEntity.setTo(command.getTo()); + commandEntity.setContent(command.getContent()); + if (command.getTags() != null && !command.getTags().isEmpty()) { + commandEntity.setTags( + command.getTags() + .stream() + .map(CommandTags::valueOf) + .collect(Collectors.toList())); + } + + return commandEntity; + } +} diff --git a/gravitee-management-api-service/src/main/java/io/gravitee/management/service/impl/MessageServiceImpl.java b/gravitee-management-api-service/src/main/java/io/gravitee/management/service/impl/MessageServiceImpl.java index 3cccef3f21..dddcdd9898 100644 --- a/gravitee-management-api-service/src/main/java/io/gravitee/management/service/impl/MessageServiceImpl.java +++ b/gravitee-management-api-service/src/main/java/io/gravitee/management/service/impl/MessageServiceImpl.java @@ -48,7 +48,7 @@ import java.util.*; import java.util.stream.Collectors; -import static io.gravitee.management.service.impl.MessageServiceImpl.MesssageEvent.MESSAGE_SENT; +import static io.gravitee.management.service.impl.MessageServiceImpl.MessageEvent.MESSAGE_SENT; /** * @author Nicolas GERAUD (nicolas.geraud at graviteesource.com) @@ -92,7 +92,7 @@ public class MessageServiceImpl extends AbstractService implements MessageServic @Value("${email.from}") private String defaultFrom; - public enum MesssageEvent implements Audit.AuditEvent { + public enum MessageEvent implements Audit.AuditEvent { MESSAGE_SENT } diff --git a/gravitee-management-api-service/src/main/java/io/gravitee/management/service/impl/PageServiceImpl.java b/gravitee-management-api-service/src/main/java/io/gravitee/management/service/impl/PageServiceImpl.java index 1eaed4f7d0..bf87f210ec 100644 --- a/gravitee-management-api-service/src/main/java/io/gravitee/management/service/impl/PageServiceImpl.java +++ b/gravitee-management-api-service/src/main/java/io/gravitee/management/service/impl/PageServiceImpl.java @@ -456,7 +456,7 @@ public PageEntity update(String pageId, UpdatePageEntity updatePageEntity, boole // update document in search engine if(pageToUpdate.isPublished() && !page.isPublished()) { - searchEngineService.delete(convert(pageToUpdate)); + searchEngineService.delete(convert(pageToUpdate), false); } else { index(pageEntity); } @@ -470,7 +470,7 @@ public PageEntity update(String pageId, UpdatePageEntity updatePageEntity, boole private void index(PageEntity pageEntity) { if (pageEntity.isPublished()) { - searchEngineService.index(pageEntity); + searchEngineService.index(pageEntity, false); } } @@ -558,7 +558,7 @@ public void delete(String pageId) { } // remove from search engine - searchEngineService.delete(convert(optPage.get())); + searchEngineService.delete(convert(optPage.get()), false); } catch (TechnicalException ex) { logger.error("An error occurs while trying to delete Page {}", pageId, ex); throw new TechnicalManagementException("An error occurs while trying to delete Page " + pageId, ex); diff --git a/gravitee-management-api-service/src/main/java/io/gravitee/management/service/impl/search/SearchEngineServiceImpl.java b/gravitee-management-api-service/src/main/java/io/gravitee/management/service/impl/search/SearchEngineServiceImpl.java index f55b37af10..410f617635 100644 --- a/gravitee-management-api-service/src/main/java/io/gravitee/management/service/impl/search/SearchEngineServiceImpl.java +++ b/gravitee-management-api-service/src/main/java/io/gravitee/management/service/impl/search/SearchEngineServiceImpl.java @@ -15,26 +15,37 @@ */ package io.gravitee.management.service.impl.search; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.gravitee.management.model.ApiPageEntity; +import io.gravitee.management.model.PageEntity; +import io.gravitee.management.model.api.ApiEntity; +import io.gravitee.management.model.command.CommandSearchIndexerEntity; +import io.gravitee.management.model.command.CommandTags; +import io.gravitee.management.model.command.NewCommandEntity; import io.gravitee.management.model.search.Indexable; +import io.gravitee.management.service.ApiService; +import io.gravitee.management.service.CommandService; +import io.gravitee.management.service.PageService; +import io.gravitee.management.service.exceptions.TechnicalManagementException; import io.gravitee.management.service.impl.search.lucene.DocumentSearcher; import io.gravitee.management.service.impl.search.lucene.DocumentTransformer; import io.gravitee.management.service.impl.search.lucene.SearchEngineIndexer; import io.gravitee.management.service.search.SearchEngineService; import io.gravitee.repository.exceptions.TechnicalException; +import io.gravitee.repository.management.model.MessageRecipient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Component; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Optional; +import java.util.*; import java.util.stream.Collectors; /** * @author David BRASSELY (david.brassely at graviteesource.com) + * @author Nicolas GERAUD (nicolas.geraud at graviteesource.com) * @author GraviteeSource Team */ @Component @@ -54,9 +65,88 @@ public class SearchEngineServiceImpl implements SearchEngineService { @Autowired private Collection searchers; + @Autowired + private CommandService commandService; + + @Autowired + private ApiService apiService; + + @Autowired + private PageService pageService; + + private ObjectMapper mapper = new ObjectMapper(); + + private static final String ACTION_INDEX = "I"; + private static final String ACTION_DELETE = "D"; + @Async @Override - public void index(Indexable source) { + public void index(Indexable source, boolean locally) { + if (locally) { + indexLocally(source); + } else { + CommandSearchIndexerEntity content = new CommandSearchIndexerEntity(); + content.setAction(ACTION_INDEX); + content.setId(source.getId()); + content.setClazz(source.getClass().getName()); + + sendCommands(content); + } + } + + @Async + @Override + public void delete(Indexable source, boolean locally) { + if (locally) { + deleteLocally(source); + } else { + CommandSearchIndexerEntity content = new CommandSearchIndexerEntity(); + content.setAction(ACTION_DELETE); + content.setId(source.getId()); + content.setClazz(source.getClass().getName()); + + sendCommands(content); + } + } + + private void sendCommands(CommandSearchIndexerEntity content) { + try { + NewCommandEntity msg = new NewCommandEntity(); + msg.setTags(Collections.singletonList(CommandTags.DATA_TO_INDEX)); + msg.setTo(MessageRecipient.MANAGEMENT_APIS.name()); + msg.setTtlInSeconds(60); + msg.setContent(mapper.writeValueAsString(content)); + commandService.send(msg); + } catch (JsonProcessingException e) { + logger.error("Unexpected error while sending a message", e); + } + } + + @Override + public void process(CommandSearchIndexerEntity content) { + Indexable source = getSource(content.getClazz(), content.getId()); + if (source == null) { + logger.error("Unable to get source from message content [{}]", content); + throw new TechnicalManagementException("Unable to get source from message content [" + content + "]"); + } + + if (ACTION_DELETE.equals(content.getAction())) { + deleteLocally(source); + } else if (ACTION_INDEX.equals(content.getAction())) { + indexLocally(source); + } + } + + private Indexable getSource(String clazz, String id) { + if (ApiEntity.class.getName().equals(clazz)) { + return apiService.findById(id); + } else if (PageEntity.class.getName().equals(clazz) || ApiPageEntity.class.getName().equals(clazz)) { + return pageService.findById(id); + } + return null; + } + + private void indexLocally(Indexable source) { transformers.stream() .filter(transformer -> transformer.handle(source.getClass())) .findFirst() @@ -69,9 +159,7 @@ public void index(Indexable source) { }); } - @Async - @Override - public void delete(Indexable source) { + private void deleteLocally(Indexable source) { transformers.stream() .filter(transformer -> transformer.handle(source.getClass())) .findFirst() @@ -104,4 +192,5 @@ public Collection search(io.gravitee.management.service.search.query.Que return results.get(); } + } diff --git a/gravitee-management-api-service/src/main/java/io/gravitee/management/service/impl/upgrade/SearchIndexUpgrader.java b/gravitee-management-api-service/src/main/java/io/gravitee/management/service/impl/upgrade/SearchIndexUpgrader.java index b92dba543c..30626e9ea8 100644 --- a/gravitee-management-api-service/src/main/java/io/gravitee/management/service/impl/upgrade/SearchIndexUpgrader.java +++ b/gravitee-management-api-service/src/main/java/io/gravitee/management/service/impl/upgrade/SearchIndexUpgrader.java @@ -54,13 +54,13 @@ public boolean upgrade() { Set apis = apiService.findAll(); apis.stream() .forEach(apiEntity -> { - searchEngineService.index(apiEntity); + searchEngineService.index(apiEntity, true); List apiPages = pageService.search(new PageQuery.Builder().api(apiEntity.getId()).published(true).build()); apiPages.stream().forEach(pageListItem -> { try { PageEntity page = pageService.findById(pageListItem.getId(), true); - searchEngineService.index(page); + searchEngineService.index(page, true); } catch (Exception ex) { } diff --git a/gravitee-management-api-service/src/main/java/io/gravitee/management/service/search/SearchEngineService.java b/gravitee-management-api-service/src/main/java/io/gravitee/management/service/search/SearchEngineService.java index b13776b8a7..78622bb76d 100644 --- a/gravitee-management-api-service/src/main/java/io/gravitee/management/service/search/SearchEngineService.java +++ b/gravitee-management-api-service/src/main/java/io/gravitee/management/service/search/SearchEngineService.java @@ -15,6 +15,7 @@ */ package io.gravitee.management.service.search; +import io.gravitee.management.model.command.CommandSearchIndexerEntity; import io.gravitee.management.model.search.Indexable; import io.gravitee.management.service.search.query.Query; @@ -26,9 +27,11 @@ */ public interface SearchEngineService { - void index(Indexable source); + void index(Indexable source, boolean locally); - void delete(Indexable source); + void delete(Indexable source, boolean locally); Collection search(Query query); + + void process(CommandSearchIndexerEntity content); } diff --git a/gravitee-management-api-services/gravitee-management-api-services-search-indexer/pom.xml b/gravitee-management-api-services/gravitee-management-api-services-search-indexer/pom.xml new file mode 100644 index 0000000000..d8bb4fdcbf --- /dev/null +++ b/gravitee-management-api-services/gravitee-management-api-services-search-indexer/pom.xml @@ -0,0 +1,85 @@ + + + + 4.0.0 + + + io.gravitee.management.services + gravitee-management-api-services + 1.20.16-SNAPSHOT + + + + gravitee-management-api-services-search-indexer + Gravitee.io APIM - Management - Services - Search Indexer + + + + + org.springframework + spring-core + ${spring.version} + provided + + + org.springframework + spring-context + ${spring.version} + provided + + + commons-logging + commons-logging + + + + + + + + + src/main/resources + true + + + + + maven-assembly-plugin + 2.3 + + false + + src/main/assembly/plugin-assembly.xml + + + + + make-plugin-assembly + package + + single + + + + + + + diff --git a/gravitee-management-api-services/gravitee-management-api-services-search-indexer/src/main/assembly/plugin-assembly.xml b/gravitee-management-api-services/gravitee-management-api-services-search-indexer/src/main/assembly/plugin-assembly.xml new file mode 100644 index 0000000000..03512e179e --- /dev/null +++ b/gravitee-management-api-services/gravitee-management-api-services-search-indexer/src/main/assembly/plugin-assembly.xml @@ -0,0 +1,40 @@ + + + + plugin + + zip + + false + + + + + ${project.build.directory}/${project.build.finalName}.jar + + + + + + + lib + false + + + \ No newline at end of file diff --git a/gravitee-management-api-services/gravitee-management-api-services-search-indexer/src/main/java/io/gravitee/management/services/search/ScheduledSearchIndexerService.java b/gravitee-management-api-services/gravitee-management-api-services-search-indexer/src/main/java/io/gravitee/management/services/search/ScheduledSearchIndexerService.java new file mode 100644 index 0000000000..298e2f3783 --- /dev/null +++ b/gravitee-management-api-services/gravitee-management-api-services-search-indexer/src/main/java/io/gravitee/management/services/search/ScheduledSearchIndexerService.java @@ -0,0 +1,105 @@ +/** + * Copyright (C) 2015 The Gravitee team (http://gravitee.io) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.gravitee.management.services.search; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.gravitee.common.service.AbstractService; +import io.gravitee.management.model.command.CommandEntity; +import io.gravitee.management.model.command.CommandQuery; +import io.gravitee.management.model.command.CommandSearchIndexerEntity; +import io.gravitee.management.model.command.CommandTags; +import io.gravitee.management.service.CommandService; +import io.gravitee.management.service.search.SearchEngineService; +import io.gravitee.repository.management.model.MessageRecipient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.TaskScheduler; +import org.springframework.scheduling.support.CronTrigger; + +import java.io.IOException; +import java.time.Instant; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; + +/** + * @author Nicolas GERAUD (nicolas.geraud at graviteesource.com) + * @author GraviteeSource Team + */ +public class ScheduledSearchIndexerService extends AbstractService implements Runnable { + + /** + * Logger. + */ + private final Logger logger = LoggerFactory.getLogger(ScheduledSearchIndexerService.class); + + @Autowired + private TaskScheduler scheduler; + + @Value("${services.search_indexer.cron:*/5 * * * * *}") + private String cronTrigger; + + @Value("${services.search_indexer.enabled:true}") + private boolean enabled; + + private final AtomicLong counter = new AtomicLong(0); + + @Autowired + private CommandService commandService; + + @Autowired + private SearchEngineService searchEngineService; + + private ObjectMapper mapper = new ObjectMapper(); + + @Override + protected String name() { + return "Search Indexer Service"; + } + + @Override + protected void doStart() throws Exception { + if (enabled) { + super.doStart(); + logger.info("Search Indexer service has been initialized with cron [{}]", cronTrigger); + scheduler.schedule(this, new CronTrigger(cronTrigger)); + } else { + logger.warn("Search Indexer service has been disabled"); + } + } + + @Override + public void run() { + logger.debug("Search Indexer #{} started at {}", counter.incrementAndGet(), Instant.now()); + CommandQuery query = new CommandQuery(); + query.setTo(MessageRecipient.MANAGEMENT_APIS.name()); + query.setTags(Collections.singletonList(CommandTags.DATA_TO_INDEX)); + List messageEntities = commandService.search(query); + messageEntities.forEach(commandEntity -> { + commandService.ack(commandEntity.getId()); + try { + searchEngineService.process( + mapper.readValue(commandEntity.getContent(), CommandSearchIndexerEntity.class)); + } catch (IOException e) { + logger.error("Search Indexer has received a bad message.", e); + } + }); + + logger.debug("Search Indexer #{} ended at {}", counter.get(), Instant.now()); + } +} diff --git a/gravitee-management-api-services/gravitee-management-api-services-search-indexer/src/main/java/io/gravitee/management/services/search/spring/SearchIndexerConfiguration.java b/gravitee-management-api-services/gravitee-management-api-services-search-indexer/src/main/java/io/gravitee/management/services/search/spring/SearchIndexerConfiguration.java new file mode 100644 index 0000000000..1b992e4be8 --- /dev/null +++ b/gravitee-management-api-services/gravitee-management-api-services-search-indexer/src/main/java/io/gravitee/management/services/search/spring/SearchIndexerConfiguration.java @@ -0,0 +1,36 @@ +/** + * Copyright (C) 2015 The Gravitee team (http://gravitee.io) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.gravitee.management.services.search.spring; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.TaskScheduler; +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; + +/** + * @author Nicolas GERAUD (nicolas.geraud at graviteesource.com) + * @author GraviteeSource Team + */ +@Configuration +public class SearchIndexerConfiguration { + + @Bean + public TaskScheduler taskScheduler() { + ThreadPoolTaskScheduler scheduler = new ThreadPoolTaskScheduler(); + scheduler.setThreadNamePrefix("searchindexer-"); + return scheduler; + } +} diff --git a/gravitee-management-api-services/gravitee-management-api-services-search-indexer/src/main/resources/plugin.properties b/gravitee-management-api-services/gravitee-management-api-services-search-indexer/src/main/resources/plugin.properties new file mode 100644 index 0000000000..901328a97f --- /dev/null +++ b/gravitee-management-api-services/gravitee-management-api-services-search-indexer/src/main/resources/plugin.properties @@ -0,0 +1,22 @@ +# +# Copyright (C) 2015 The Gravitee team (http://gravitee.io) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +id=search-indexer +name=${project.name} +version=${project.version} +description=${project.description} +class=io.gravitee.management.services.search.ScheduledSearchIndexerService +type=service diff --git a/gravitee-management-api-services/gravitee-management-api-services-search-indexer/src/test/java/io/gravitee/management/services/search/ScheduledSearchIndexerServiceTest.java b/gravitee-management-api-services/gravitee-management-api-services-search-indexer/src/test/java/io/gravitee/management/services/search/ScheduledSearchIndexerServiceTest.java new file mode 100644 index 0000000000..8c0d0ea3a5 --- /dev/null +++ b/gravitee-management-api-services/gravitee-management-api-services-search-indexer/src/test/java/io/gravitee/management/services/search/ScheduledSearchIndexerServiceTest.java @@ -0,0 +1,77 @@ +/** + * Copyright (C) 2015 The Gravitee team (http://gravitee.io) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.gravitee.management.services.search; + +import io.gravitee.management.model.command.CommandEntity; +import io.gravitee.management.model.command.CommandTags; +import io.gravitee.management.service.CommandService; +import io.gravitee.management.service.search.SearchEngineService; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; + +import java.util.Arrays; +import java.util.Collections; + +import static org.mockito.Mockito.*; + +/** + * @author Nicolas GERAUD (nicolas.geraud at graviteesource.com) + * @author GraviteeSource Team + */ +@RunWith(MockitoJUnitRunner.class) +public class ScheduledSearchIndexerServiceTest { + + @InjectMocks + ScheduledSearchIndexerService service = new ScheduledSearchIndexerService(); + + @Mock + CommandService commandService; + + @Mock + SearchEngineService searchEngineService; + + @Test + public void shouldDoNothing() { + when(commandService.search(any())).thenReturn(Collections.emptyList()); + + service.run(); + + verify(commandService, never()).ack(anyString()); + verify(searchEngineService, never()).process(any()); + } + + + @Test + public void shouldInsertAndDelete() { + CommandEntity insert = new CommandEntity(); + insert.setId("insertid"); + insert.setTags(Collections.singletonList(CommandTags.DATA_TO_INDEX)); + insert.setContent("{\"id\":\"1\"}"); + CommandEntity delete = new CommandEntity(); + delete.setId("deleteid"); + delete.setTags(Collections.singletonList(CommandTags.DATA_TO_INDEX)); + delete.setContent("{\"id\":\"2\"}"); + when(commandService.search(any())).thenReturn(Arrays.asList(delete, insert)); + + service.run(); + + verify(commandService, times(2)).ack(anyString()); + verify(searchEngineService, times(2)).process(any()); + } +} diff --git a/gravitee-management-api-services/pom.xml b/gravitee-management-api-services/pom.xml index 1a049450cc..67b53f1ebd 100644 --- a/gravitee-management-api-services/pom.xml +++ b/gravitee-management-api-services/pom.xml @@ -37,6 +37,7 @@ gravitee-management-api-services-dynamic-properties gravitee-management-api-services-subscriptions gravitee-management-api-services-dictionary + gravitee-management-api-services-search-indexer diff --git a/pom.xml b/pom.xml index 365d4b60d9..41bd9983ff 100644 --- a/pom.xml +++ b/pom.xml @@ -38,7 +38,7 @@ 1.13.3 1.13.0 1.5.0 - 1.20.2 + 1.20.3-SNAPSHOT 1.11.0 1.0.0 1.0.0