From 7de6f8b2c0d59829de3bd93b642f65a81ae3b097 Mon Sep 17 00:00:00 2001 From: David BRASSELY Date: Wed, 31 Jul 2019 19:43:25 +0200 Subject: [PATCH] feat(elasticsearch): Add ES 7.x support Closes gravitee-io/issues#2165 --- .../gravitee/elasticsearch/client/Client.java | 3 +- .../elasticsearch/client/http/HttpClient.java | 18 +-- .../elasticsearch/model/SearchHits.java | 7 +- .../elasticsearch/model/TotalHits.java | 40 +++++ .../model/jackson/TotalHitsDeserializer.java | 54 +++++++ .../io/gravitee/elasticsearch/utils/Type.java | 5 +- .../version/ElasticsearchInfo.java | 71 +++++++++ .../elasticsearch/version/Version.java | 59 ++++++++ .../elasticsearch/client/HttpClientTest.java | 5 +- .../elasticsearch/ElasticsearchReporter.java | 11 +- .../indexer/es7/ES7BulkIndexer.java | 45 ++++++ .../mapping/es7/ES7IndexPreparer.java | 73 +++++++++ .../context/Elastic7xBeanRegistrer.java | 40 +++++ .../freemarker/es7x/index/health.ftl | 66 +++++++++ .../resources/freemarker/es7x/index/log.ftl | 99 +++++++++++++ .../freemarker/es7x/index/monitor.ftl | 69 +++++++++ .../freemarker/es7x/index/request.ftl | 67 +++++++++ .../es7x/mapping/index-template-health.ftl | 44 ++++++ .../es7x/mapping/index-template-log.ftl | 51 +++++++ .../es7x/mapping/index-template-monitor.ftl | 19 +++ .../es7x/mapping/index-template-request.ftl | 139 ++++++++++++++++++ .../AbstractElasticsearchRepository.java | 4 + .../AbstractElasticsearchQueryCommand.java | 8 +- .../analytics/query/CountQueryCommand.java | 2 +- .../ElasticsearchHealthCheckRepository.java | 4 +- .../AbstractElasticsearchQueryCommand.java | 4 + .../query/AverageAvailabilityCommand.java | 2 +- .../query/AverageDateHistogramCommand.java | 2 +- .../query/AverageResponseTimeCommand.java | 2 +- .../healthcheck/query/LogsCommand.java | 4 +- .../log/ElasticLogRepository.java | 16 +- .../ElasticsearchMonitoringRepository.java | 2 +- .../ElasticsearchRepositoryConfiguration.java | 22 +-- .../src/main/resources/freemarker/groupBy.ftl | 4 +- pom.xml | 52 +++++++ 35 files changed, 1057 insertions(+), 56 deletions(-) create mode 100644 gravitee-common-elasticsearch/src/main/java/io/gravitee/elasticsearch/model/TotalHits.java create mode 100644 gravitee-common-elasticsearch/src/main/java/io/gravitee/elasticsearch/model/jackson/TotalHitsDeserializer.java create mode 100644 gravitee-common-elasticsearch/src/main/java/io/gravitee/elasticsearch/version/ElasticsearchInfo.java create mode 100644 gravitee-common-elasticsearch/src/main/java/io/gravitee/elasticsearch/version/Version.java create mode 100644 gravitee-reporter-elasticsearch/src/main/java/io/gravitee/reporter/elasticsearch/indexer/es7/ES7BulkIndexer.java create mode 100644 gravitee-reporter-elasticsearch/src/main/java/io/gravitee/reporter/elasticsearch/mapping/es7/ES7IndexPreparer.java create mode 100644 gravitee-reporter-elasticsearch/src/main/java/io/gravitee/reporter/elasticsearch/spring/context/Elastic7xBeanRegistrer.java create mode 100644 gravitee-reporter-elasticsearch/src/main/resources/freemarker/es7x/index/health.ftl create mode 100644 gravitee-reporter-elasticsearch/src/main/resources/freemarker/es7x/index/log.ftl create mode 100644 gravitee-reporter-elasticsearch/src/main/resources/freemarker/es7x/index/monitor.ftl create mode 100644 gravitee-reporter-elasticsearch/src/main/resources/freemarker/es7x/index/request.ftl create mode 100644 gravitee-reporter-elasticsearch/src/main/resources/freemarker/es7x/mapping/index-template-health.ftl create mode 100644 gravitee-reporter-elasticsearch/src/main/resources/freemarker/es7x/mapping/index-template-log.ftl create mode 100644 gravitee-reporter-elasticsearch/src/main/resources/freemarker/es7x/mapping/index-template-monitor.ftl create mode 100644 gravitee-reporter-elasticsearch/src/main/resources/freemarker/es7x/mapping/index-template-request.ftl diff --git a/gravitee-common-elasticsearch/src/main/java/io/gravitee/elasticsearch/client/Client.java b/gravitee-common-elasticsearch/src/main/java/io/gravitee/elasticsearch/client/Client.java index 4c7abdc0..ec9f0faa 100644 --- a/gravitee-common-elasticsearch/src/main/java/io/gravitee/elasticsearch/client/Client.java +++ b/gravitee-common-elasticsearch/src/main/java/io/gravitee/elasticsearch/client/Client.java @@ -19,6 +19,7 @@ import io.gravitee.elasticsearch.model.Health; import io.gravitee.elasticsearch.model.SearchResponse; import io.gravitee.elasticsearch.model.bulk.BulkResponse; +import io.gravitee.elasticsearch.version.ElasticsearchInfo; import io.reactivex.Completable; import io.reactivex.Single; import io.vertx.core.buffer.Buffer; @@ -31,7 +32,7 @@ */ public interface Client { - Single getVersion() throws ElasticsearchException; + Single getInfo() throws ElasticsearchException; Single getClusterHealth(); diff --git a/gravitee-common-elasticsearch/src/main/java/io/gravitee/elasticsearch/client/http/HttpClient.java b/gravitee-common-elasticsearch/src/main/java/io/gravitee/elasticsearch/client/http/HttpClient.java index 935dceec..5c578954 100644 --- a/gravitee-common-elasticsearch/src/main/java/io/gravitee/elasticsearch/client/http/HttpClient.java +++ b/gravitee-common-elasticsearch/src/main/java/io/gravitee/elasticsearch/client/http/HttpClient.java @@ -24,12 +24,10 @@ import io.gravitee.elasticsearch.exception.ElasticsearchException; import io.gravitee.elasticsearch.model.Health; import io.gravitee.elasticsearch.model.SearchResponse; -import io.gravitee.elasticsearch.model.bulk.BulkItemResponse; import io.gravitee.elasticsearch.model.bulk.BulkResponse; +import io.gravitee.elasticsearch.version.ElasticsearchInfo; import io.reactivex.Completable; -import io.reactivex.Flowable; import io.reactivex.Single; -import io.reactivex.functions.Function; import io.vertx.core.Handler; import io.vertx.ext.web.client.WebClientOptions; import io.vertx.ext.web.client.impl.HttpContext; @@ -46,8 +44,6 @@ import java.nio.charset.StandardCharsets; import java.util.Base64; import java.util.List; -import java.util.function.Consumer; -import java.util.function.Predicate; /** * @author David BRASSELY (david.brassely at graviteesource.com) @@ -143,20 +139,12 @@ public void handle(HttpContext context) { } @Override - public Single getVersion() throws ElasticsearchException { + public Single getInfo() throws ElasticsearchException { return httpClient .get(URL_ROOT) .rxSend() .doOnError(throwable -> logger.error("Unable to get a connection to Elasticsearch", throwable)) - .map(response -> mapper.readTree(response.bodyAsString()).path("version").path("number").asText()) - .map(sVersion -> { - float result = Float.valueOf(sVersion.substring(0, 3)); - int version = Integer.valueOf(sVersion.substring(0, 1)); - if (result < 2) { - logger.warn("Please upgrade to Elasticsearch 2 or later. version={}", version); - } - return version; - }); + .map(response -> mapper.readValue(response.bodyAsString(), ElasticsearchInfo.class)); } /** diff --git a/gravitee-common-elasticsearch/src/main/java/io/gravitee/elasticsearch/model/SearchHits.java b/gravitee-common-elasticsearch/src/main/java/io/gravitee/elasticsearch/model/SearchHits.java index 3032701a..a97731d9 100644 --- a/gravitee-common-elasticsearch/src/main/java/io/gravitee/elasticsearch/model/SearchHits.java +++ b/gravitee-common-elasticsearch/src/main/java/io/gravitee/elasticsearch/model/SearchHits.java @@ -40,7 +40,8 @@ public class SearchHits implements Serializable { /** * The total number of hits that matches the search request. */ - private Long total; + @JsonProperty("total") + private TotalHits total; /** * The maximum score of this query. @@ -53,11 +54,11 @@ public class SearchHits implements Serializable { */ private List hits; - public Long getTotal() { + public TotalHits getTotal() { return total; } - public void setTotal(Long total) { + public void setTotal(TotalHits total) { this.total = total; } diff --git a/gravitee-common-elasticsearch/src/main/java/io/gravitee/elasticsearch/model/TotalHits.java b/gravitee-common-elasticsearch/src/main/java/io/gravitee/elasticsearch/model/TotalHits.java new file mode 100644 index 00000000..c78a4e34 --- /dev/null +++ b/gravitee-common-elasticsearch/src/main/java/io/gravitee/elasticsearch/model/TotalHits.java @@ -0,0 +1,40 @@ +/** + * Copyright (C) 2015 The Gravitee team (http://gravitee.io) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.gravitee.elasticsearch.model; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import io.gravitee.elasticsearch.model.jackson.TotalHitsDeserializer; + +/** + * @author David BRASSELY (david.brassely at graviteesource.com) + * @author GraviteeSource Team + */ +@JsonDeserialize(using = TotalHitsDeserializer.class) +public class TotalHits { + private long value; + + public TotalHits(final long value) { + this.value = value; + } + + public long getValue() { + return value; + } + + public void setValue(long value) { + this.value = value; + } +} diff --git a/gravitee-common-elasticsearch/src/main/java/io/gravitee/elasticsearch/model/jackson/TotalHitsDeserializer.java b/gravitee-common-elasticsearch/src/main/java/io/gravitee/elasticsearch/model/jackson/TotalHitsDeserializer.java new file mode 100644 index 00000000..5958f997 --- /dev/null +++ b/gravitee-common-elasticsearch/src/main/java/io/gravitee/elasticsearch/model/jackson/TotalHitsDeserializer.java @@ -0,0 +1,54 @@ +/** + * Copyright (C) 2015 The Gravitee team (http://gravitee.io) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.gravitee.elasticsearch.model.jackson; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; +import io.gravitee.elasticsearch.model.TotalHits; + +import java.io.IOException; + +/** + * @author David BRASSELY (david.brassely at graviteesource.com) + * @author GraviteeSource Team + */ +public class TotalHitsDeserializer extends StdDeserializer { + + public TotalHitsDeserializer() { + this(null); + } + + public TotalHitsDeserializer(Class vc) { + super(vc); + } + + @Override + public TotalHits deserialize(JsonParser jp, DeserializationContext ctxt) + throws IOException { + JsonNode node = jp.getCodec().readTree(jp); + + if (node.isNumber()) { + return new TotalHits(node.longValue()); + } else if (node.isObject()) { + return new TotalHits(node.get("value").longValue()); + } + + throw new IllegalStateException(); + } +} + diff --git a/gravitee-common-elasticsearch/src/main/java/io/gravitee/elasticsearch/utils/Type.java b/gravitee-common-elasticsearch/src/main/java/io/gravitee/elasticsearch/utils/Type.java index eb63a601..dc2b9bc2 100644 --- a/gravitee-common-elasticsearch/src/main/java/io/gravitee/elasticsearch/utils/Type.java +++ b/gravitee-common-elasticsearch/src/main/java/io/gravitee/elasticsearch/utils/Type.java @@ -24,7 +24,10 @@ public enum Type { REQUEST("request"), HEALTH_CHECK("health"), LOG("log"), - MONITOR("monitor"); + MONITOR("monitor"), + + // For ES7 support + DOC("_doc"); private String type; diff --git a/gravitee-common-elasticsearch/src/main/java/io/gravitee/elasticsearch/version/ElasticsearchInfo.java b/gravitee-common-elasticsearch/src/main/java/io/gravitee/elasticsearch/version/ElasticsearchInfo.java new file mode 100644 index 00000000..da0c6246 --- /dev/null +++ b/gravitee-common-elasticsearch/src/main/java/io/gravitee/elasticsearch/version/ElasticsearchInfo.java @@ -0,0 +1,71 @@ +/** + * Copyright (C) 2015 The Gravitee team (http://gravitee.io) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.gravitee.elasticsearch.version; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author David BRASSELY (david.brassely at graviteesource.com) + * @author GraviteeSource Team + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class ElasticsearchInfo { + + private String name; + + @JsonProperty("cluster_name") + private String clusterName; + + @JsonProperty("cluster_uuid") + private String clusterUuid; + + private Version version; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getClusterName() { + return clusterName; + } + + public void setClusterName(String clusterName) { + this.clusterName = clusterName; + } + + public String getClusterUuid() { + return clusterUuid; + } + + public void setClusterUuid(String clusterUuid) { + this.clusterUuid = clusterUuid; + } + + public Version getVersion() { + return version; + } + + public void setVersion(Version version) { + this.version = version; + } +} diff --git a/gravitee-common-elasticsearch/src/main/java/io/gravitee/elasticsearch/version/Version.java b/gravitee-common-elasticsearch/src/main/java/io/gravitee/elasticsearch/version/Version.java new file mode 100644 index 00000000..9014c215 --- /dev/null +++ b/gravitee-common-elasticsearch/src/main/java/io/gravitee/elasticsearch/version/Version.java @@ -0,0 +1,59 @@ +/** + * Copyright (C) 2015 The Gravitee team (http://gravitee.io) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.gravitee.elasticsearch.version; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author David BRASSELY (david.brassely at graviteesource.com) + * @author GraviteeSource Team + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class Version { + + private String number; + + @JsonProperty("lucene_version") + private String luceneVersion; + + private int majorVersion = -1; + + public String getNumber() { + return number; + } + + public void setNumber(String number) { + this.number = number; + } + + public String getLuceneVersion() { + return luceneVersion; + } + + public void setLuceneVersion(String luceneVersion) { + this.luceneVersion = luceneVersion; + } + + public int getMajorVersion() { + if (majorVersion == -1) { + majorVersion = Integer.valueOf(getNumber().substring(0, 1)); + } + return majorVersion; + } +} diff --git a/gravitee-common-elasticsearch/src/test/java/io/gravitee/elasticsearch/client/HttpClientTest.java b/gravitee-common-elasticsearch/src/test/java/io/gravitee/elasticsearch/client/HttpClientTest.java index 31e95d85..8ecd64e6 100644 --- a/gravitee-common-elasticsearch/src/test/java/io/gravitee/elasticsearch/client/HttpClientTest.java +++ b/gravitee-common-elasticsearch/src/test/java/io/gravitee/elasticsearch/client/HttpClientTest.java @@ -20,6 +20,7 @@ import io.gravitee.elasticsearch.config.Endpoint; import io.gravitee.elasticsearch.embedded.ElasticsearchNode; import io.gravitee.elasticsearch.model.Health; +import io.gravitee.elasticsearch.version.ElasticsearchInfo; import io.reactivex.Single; import io.reactivex.observers.TestObserver; import io.vertx.reactivex.core.Vertx; @@ -61,9 +62,9 @@ public class HttpClientTest { @Test public void shouldGetVersion() throws InterruptedException, ExecutionException, IOException { - Single version = client.getVersion(); + Single info = client.getInfo(); - TestObserver observer = version.test(); + TestObserver observer = info.test(); observer.awaitTerminalEvent(); observer.assertNoErrors(); diff --git a/gravitee-reporter-elasticsearch/src/main/java/io/gravitee/reporter/elasticsearch/ElasticsearchReporter.java b/gravitee-reporter-elasticsearch/src/main/java/io/gravitee/reporter/elasticsearch/ElasticsearchReporter.java index ba9f42b8..49e4c6cf 100644 --- a/gravitee-reporter-elasticsearch/src/main/java/io/gravitee/reporter/elasticsearch/ElasticsearchReporter.java +++ b/gravitee-reporter-elasticsearch/src/main/java/io/gravitee/reporter/elasticsearch/ElasticsearchReporter.java @@ -17,6 +17,7 @@ import io.gravitee.common.service.AbstractService; import io.gravitee.elasticsearch.client.Client; +import io.gravitee.elasticsearch.version.ElasticsearchInfo; import io.gravitee.reporter.api.Reportable; import io.gravitee.reporter.api.Reporter; import io.gravitee.reporter.api.health.EndpointStatus; @@ -29,6 +30,7 @@ import io.gravitee.reporter.elasticsearch.spring.context.Elastic2xBeanRegistrer; import io.gravitee.reporter.elasticsearch.spring.context.Elastic5xBeanRegistrer; import io.gravitee.reporter.elasticsearch.spring.context.Elastic6xBeanRegistrer; +import io.gravitee.reporter.elasticsearch.spring.context.Elastic7xBeanRegistrer; import io.reactivex.BackpressureStrategy; import io.reactivex.CompletableObserver; import io.reactivex.Observable; @@ -68,19 +70,19 @@ protected void doStart() throws Exception { logger.info("Starting Elastic reporter engine..."); // Wait for a connection to ES and retry each 5 seconds - Single singleVersion = client.getVersion() + Single singleVersion = client.getInfo() .retryWhen(error -> error.flatMap( throwable -> Observable.just(new Object()).delay(5, TimeUnit.SECONDS).toFlowable(BackpressureStrategy.LATEST))); singleVersion.subscribe(); - Integer version = singleVersion.blockingGet(); + ElasticsearchInfo version = singleVersion.blockingGet(); boolean registered = true; DefaultListableBeanFactory beanFactory = (DefaultListableBeanFactory) applicationContext.getAutowireCapableBeanFactory(); - switch (version) { + switch (version.getVersion().getMajorVersion()) { case 2: new Elastic2xBeanRegistrer().register(beanFactory, configuration.isPerTypeIndex()); break; @@ -90,6 +92,9 @@ protected void doStart() throws Exception { case 6: new Elastic6xBeanRegistrer().register(beanFactory); break; + case 7: + new Elastic7xBeanRegistrer().register(beanFactory); + break; default: registered = false; logger.error("Version {} is not supported by this Elasticsearch connector", version); diff --git a/gravitee-reporter-elasticsearch/src/main/java/io/gravitee/reporter/elasticsearch/indexer/es7/ES7BulkIndexer.java b/gravitee-reporter-elasticsearch/src/main/java/io/gravitee/reporter/elasticsearch/indexer/es7/ES7BulkIndexer.java new file mode 100644 index 00000000..d60cde96 --- /dev/null +++ b/gravitee-reporter-elasticsearch/src/main/java/io/gravitee/reporter/elasticsearch/indexer/es7/ES7BulkIndexer.java @@ -0,0 +1,45 @@ +/** + * Copyright (C) 2015 The Gravitee team (http://gravitee.io) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.gravitee.reporter.elasticsearch.indexer.es7; + +import io.gravitee.reporter.elasticsearch.indexer.BulkIndexer; +import io.vertx.core.buffer.Buffer; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.util.Map; + +/** + * @author David BRASSELY (david.brassely at graviteesource.com) + * @author GraviteeSource Team + */ +public class ES7BulkIndexer extends BulkIndexer { + + @Override + protected Buffer generateData(String templateName, Map data) { + try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) { + freeMarkerComponent.generateFromTemplate( + "/es7x/index/" + templateName, + data, + new OutputStreamWriter(baos)); + + return Buffer.buffer(baos.toByteArray()); + } catch (IOException e) { + return null; + } + } +} diff --git a/gravitee-reporter-elasticsearch/src/main/java/io/gravitee/reporter/elasticsearch/mapping/es7/ES7IndexPreparer.java b/gravitee-reporter-elasticsearch/src/main/java/io/gravitee/reporter/elasticsearch/mapping/es7/ES7IndexPreparer.java new file mode 100644 index 00000000..5ef099e9 --- /dev/null +++ b/gravitee-reporter-elasticsearch/src/main/java/io/gravitee/reporter/elasticsearch/mapping/es7/ES7IndexPreparer.java @@ -0,0 +1,73 @@ +/** + * Copyright (C) 2015 The Gravitee team (http://gravitee.io) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.gravitee.reporter.elasticsearch.mapping.es7; + +import io.gravitee.elasticsearch.utils.Type; +import io.gravitee.reporter.elasticsearch.config.PipelineConfiguration; +import io.gravitee.reporter.elasticsearch.mapping.PerTypeIndexPreparer; +import io.reactivex.Completable; +import io.reactivex.CompletableSource; +import io.reactivex.functions.Function; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.Map; + +/** + * @author David BRASSELY (david.brassely at graviteesource.com) + * @author GraviteeSource Team + */ +public class ES7IndexPreparer extends PerTypeIndexPreparer { + + /** + * Configuration of pipelineConfiguration + */ + @Autowired + private PipelineConfiguration pipelineConfiguration; + + @Override + public Completable prepare() { + return indexMapping().andThen(pipeline()); + } + + @Override + protected Function indexTypeMapper() { + return type -> { + final String typeName = type.getType(); + final String templateName = configuration.getIndexName() + '-' + typeName; + + logger.debug("Trying to put template mapping for type[{}] name[{}]", typeName, templateName); + + Map data = getTemplateData(); + data.put("indexName", configuration.getIndexName() + '-' + typeName); + + final String template = freeMarkerComponent.generateFromTemplate( + "/es7x/mapping/index-template-" + typeName + ".ftl", data); + + return client.putTemplate(templateName, template); + }; + } + + private Completable pipeline() { + String pipelineTemplate = pipelineConfiguration.createPipeline(); + + if (pipelineTemplate != null && pipelineConfiguration.getPipelineName() != null) { + return client.putPipeline(pipelineConfiguration.getPipelineName(), pipelineTemplate) + .doOnComplete(() -> pipelineConfiguration.valid()); + } + + return Completable.complete(); + } +} diff --git a/gravitee-reporter-elasticsearch/src/main/java/io/gravitee/reporter/elasticsearch/spring/context/Elastic7xBeanRegistrer.java b/gravitee-reporter-elasticsearch/src/main/java/io/gravitee/reporter/elasticsearch/spring/context/Elastic7xBeanRegistrer.java new file mode 100644 index 00000000..6acb75dd --- /dev/null +++ b/gravitee-reporter-elasticsearch/src/main/java/io/gravitee/reporter/elasticsearch/spring/context/Elastic7xBeanRegistrer.java @@ -0,0 +1,40 @@ +/** + * Copyright (C) 2015 The Gravitee team (http://gravitee.io) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.gravitee.reporter.elasticsearch.spring.context; + +import io.gravitee.reporter.elasticsearch.indexer.es7.ES7BulkIndexer; +import io.gravitee.reporter.elasticsearch.indexer.name.PerTypeIndexNameGenerator; +import io.gravitee.reporter.elasticsearch.mapping.es7.ES7IndexPreparer; +import org.springframework.beans.factory.support.BeanDefinitionBuilder; +import org.springframework.beans.factory.support.DefaultListableBeanFactory; + +/** + * @author David BRASSELY (david.brassely at graviteesource.com) + * @author GraviteeSource Team + */ +public class Elastic7xBeanRegistrer { + + public void register(DefaultListableBeanFactory beanFactory) { + BeanDefinitionBuilder beanIndexer = BeanDefinitionBuilder.rootBeanDefinition(ES7BulkIndexer.class); + beanFactory.registerBeanDefinition("indexer", beanIndexer.getBeanDefinition()); + + BeanDefinitionBuilder beanIndexPreparer = BeanDefinitionBuilder.rootBeanDefinition(ES7IndexPreparer.class); + beanFactory.registerBeanDefinition("indexPreparer", beanIndexPreparer.getBeanDefinition()); + + BeanDefinitionBuilder beanIndexNameGenerator = BeanDefinitionBuilder.rootBeanDefinition(PerTypeIndexNameGenerator.class); + beanFactory.registerBeanDefinition("indexNameGenerator", beanIndexNameGenerator.getBeanDefinition()); + } +} diff --git a/gravitee-reporter-elasticsearch/src/main/resources/freemarker/es7x/index/health.ftl b/gravitee-reporter-elasticsearch/src/main/resources/freemarker/es7x/index/health.ftl new file mode 100644 index 00000000..85ffc966 --- /dev/null +++ b/gravitee-reporter-elasticsearch/src/main/resources/freemarker/es7x/index/health.ftl @@ -0,0 +1,66 @@ +<#ftl output_format="JSON"> +<#macro stringOrNull data=""> + <#if data != ""> + "${data}"<#else> + null + +{ "index" : { "_index" : "${index}", "_id" : "${status.getId()}" } } +<@compress single_line=true> +{ + "gateway":"${gateway}", + "api":"${status.getApi()}", + "endpoint":"${status.getEndpoint()}", + "available":${status.isAvailable()?c}, + "response-time":${status.getResponseTime()}, + "success":${status.isSuccess()?c}, + "state":${status.getState()}, + "transition":${status.isTransition()?c}, + "steps": [ +<#list status.getSteps() as step> + {"name": "${step.getName()}", + "success":${step.isSuccess()?c}, + "request": { + "uri":"${step.getRequest().getUri()}", + "method":"${step.getRequest().getMethod()}" + <#if step.getRequest().getBody()??> + ,"body":"${step.getRequest().getBody()?j_string}" + + <#if step.getRequest().getHeaders()??> + ,"headers":{ + <#list step.getRequest().getHeaders() as headerKey, headerValue> + "${headerKey}": [ + <#list headerValue as value> + "${value?j_string}" + <#sep>, + + ] + <#sep>, + + } + + }, + "response": { + "status":${step.getResponse().getStatus()} + <#if step.getResponse().getBody()??> + ,"body":"${step.getResponse().getBody()?j_string}" + + <#if step.getResponse().getHeaders()??> + ,"headers":{ + <#list step.getResponse().getHeaders() as headerKey, headerValue> + "${headerKey}": [ + <#list headerValue as value> + "${value?j_string}" + <#sep>, + + ] + <#sep>, + + } + + }, + "response-time":${step.getResponseTime()}, + "message":<@stringOrNull data=step.getMessage()/> + }<#sep>, +], +"@timestamp":"${@timestamp}" +} diff --git a/gravitee-reporter-elasticsearch/src/main/resources/freemarker/es7x/index/log.ftl b/gravitee-reporter-elasticsearch/src/main/resources/freemarker/es7x/index/log.ftl new file mode 100644 index 00000000..8431d8a0 --- /dev/null +++ b/gravitee-reporter-elasticsearch/src/main/resources/freemarker/es7x/index/log.ftl @@ -0,0 +1,99 @@ +{ "index" : { "_index" : "${index}", "_id" : "${log.getRequestId()}" } } +<@compress single_line=true> +{ + "@timestamp":"${@timestamp}", + "api":"${log.getApi()}" + <#if log.getClientRequest()??> + ,"client-request": { + "method":"${log.getClientRequest().getMethod()}", + "uri":"${log.getClientRequest().getUri()}" + <#if log.getClientRequest().getBody()??> + ,"body":"${log.getClientRequest().getBody()?j_string}" + + <#if log.getClientRequest().getHeaders()??> + ,"headers":{ + <#list log.getClientRequest().getHeaders() as headerKey, headerValue> + "${headerKey}": [ + <#list headerValue as value> + <#if value??> + "${value?j_string}" + <#sep>, + + + ] + <#sep>, + + } + + } + ,"client-response": { + "status":${log.getClientResponse().getStatus()} + <#if log.getClientResponse().getBody()??> + ,"body":"${log.getClientResponse().getBody()?j_string}" + + <#if log.getClientResponse().getHeaders()??> + ,"headers":{ + <#list log.getClientResponse().getHeaders() as headerKey, headerValue> + "${headerKey}": [ + <#list headerValue as value> + <#if value??> + "${value?j_string}" + <#sep>, + + + ] + <#sep>, + + } + + } + + <#if log.getProxyRequest()??> + <#if log.getClientRequest()??>, + "proxy-request": { + "method":"${log.getProxyRequest().getMethod()}", + "uri":"${log.getProxyRequest().getUri()}" + <#if log.getProxyRequest().getBody()??> + ,"body":"${log.getProxyRequest().getBody()?j_string}" + + <#if log.getProxyRequest().getHeaders()??> + ,"headers":{ + <#list log.getProxyRequest().getHeaders() as headerKey, headerValue> + "${headerKey}": [ + <#list headerValue as value> + <#if value??> + "${value?j_string}" + <#sep>, + + + ] + <#sep>, + + } + + } + + <#if log.getProxyResponse()??> + ,"proxy-response": { + "status":${log.getProxyResponse().getStatus()} + <#if log.getProxyResponse().getBody()??> + ,"body":"${log.getProxyResponse().getBody()?j_string}" + + <#if log.getProxyResponse().getHeaders()??> + ,"headers":{ + <#list log.getProxyResponse().getHeaders() as headerKey, headerValue> + "${headerKey}": [ + <#list headerValue as value> + <#if value??> + "${value?j_string}" + <#sep>, + + + ] + <#sep>, + + } + + } + +} diff --git a/gravitee-reporter-elasticsearch/src/main/resources/freemarker/es7x/index/monitor.ftl b/gravitee-reporter-elasticsearch/src/main/resources/freemarker/es7x/index/monitor.ftl new file mode 100644 index 00000000..b7c7a4ed --- /dev/null +++ b/gravitee-reporter-elasticsearch/src/main/resources/freemarker/es7x/index/monitor.ftl @@ -0,0 +1,69 @@ +{ "index" : { "_index" : "${index}" } } +<@compress single_line=true> +{ + "os":{ + "cpu":{ + "percent":${percent}, + "load_average":{ + <#if load_average_1m??>"1m":${load_average_1m} + <#if load_average_5m??>,"5m":${load_average_5m} + <#if load_average_15m??>,"15m":${load_average_15m} + } + }, + "mem":{ + "total_in_bytes":${mem_total_in_bytes}, + "free_in_bytes":${mem_free_in_bytes}, + "used_in_bytes":${mem_used_in_bytes}, + "free_percent":${mem_free_percent}, + "used_percent":${mem_used_percent} + } + }, + "process":{ + "timestamp":${process_timestamp}, + "open_file_descriptors":${open_file_descriptors}, + "max_file_descriptors":${max_file_descriptors} + }, + "jvm":{ + "timestamp":${jvm_timestamp}, + "uptime_in_millis":${uptime_in_millis}, + "mem":{ + "heap_used_in_bytes":${heap_used_in_bytes}, + "heap_used_percent":${heap_used_percent}, + "heap_committed_in_bytes":${heap_committed_in_bytes}, + "heap_max_in_bytes":${heap_max_in_bytes}, + "non_heap_used_in_bytes":${non_heap_used_in_bytes}, + "non_heap_committed_in_bytes":${non_heap_committed_in_bytes}, + "pools":{ +<#list pools as pool> + "${pool.getName()}":{ + "used_in_bytes":${pool.getUsed()}, + "max_in_bytes":${pool.getMax()}, + "peak_used_in_bytes":${pool.getPeakUsed()}, + "peak_max_in_bytes":${pool.getPeakMax()} + } + <#sep>, + + } + }, + "threads":{ + "count":${count}, + "peak_count":${peak_count} + }, + "gc":{ + "collectors":{ + +<#list collectors as collector> + "${collector.getName()}":{ + "collection_count":${collector.getCollectionCount()}, + "collection_time_in_millis":${collector.getCollectionTime()} + } + <#sep>, + + + } + } + }, + "gateway":"${gateway}", + "hostname":"${hostname}", + "@timestamp":"${@timestamp}" +} diff --git a/gravitee-reporter-elasticsearch/src/main/resources/freemarker/es7x/index/request.ftl b/gravitee-reporter-elasticsearch/src/main/resources/freemarker/es7x/index/request.ftl new file mode 100644 index 00000000..3b4a18d5 --- /dev/null +++ b/gravitee-reporter-elasticsearch/src/main/resources/freemarker/es7x/index/request.ftl @@ -0,0 +1,67 @@ +{ "index" : { "_index" : "${index}", "_id" : "${metrics.getRequestId()}"<#if pipeline??>, "pipeline" : "${pipeline}"} } +<@compress single_line=true> +{ + "gateway":"${gateway}" + ,"@timestamp":"${@timestamp}" + ,"transaction":"${metrics.getTransactionId()}" + ,"method":${metrics.getHttpMethod().code()?c} + ,"uri":"${metrics.getUri()}" + ,"status":${metrics.getStatus()} + ,"response-time":${metrics.getProxyResponseTimeMs()} + <#if apiResponseTime??> + ,"api-response-time":${apiResponseTime} + + <#if proxyLatency??> + ,"proxy-latency":${proxyLatency} + + <#if requestContentLength??> + ,"request-content-length":${requestContentLength} + + <#if responseContentLength??> + ,"response-content-length":${responseContentLength} + + <#if metrics.getPlan()??> + ,"plan":"${metrics.getPlan()}" + + <#if metrics.getApi()??> + ,"api":"${metrics.getApi()}" + + <#if metrics.getApplication()??> + ,"application":"${metrics.getApplication()}" + + ,"local-address":"${metrics.getLocalAddress()}" + ,"remote-address":"${metrics.getRemoteAddress()}" + <#if metrics.getEndpoint()??> + ,"endpoint":"${metrics.getEndpoint()}" + + <#if metrics.getTenant()??> + ,"tenant":"${metrics.getTenant()}" + + <#if metrics.getMessage()??> + ,"message":"${metrics.getMessage()?j_string}" + + <#if metrics.getPath()??> + ,"path":"${metrics.getPath()}" + + <#if metrics.getMappedPath()??> + ,"mapped-path":"${metrics.getMappedPath()}" + + <#if metrics.getHost()??> + ,"host":"${metrics.getHost()}" + + <#if metrics.getUserAgent()?? || pipeline?has_content> + ,"user-agent":"${metrics.getUserAgent()!""}" + + <#if metrics.getUser()??> + ,"user":"${metrics.getUser()}" + + <#if metrics.getSecurityType()??> + ,"security-type":"${metrics.getSecurityType()}" + + <#if metrics.getSecurityToken()??> + ,"security-token":"${metrics.getSecurityToken()}" + + <#if metrics.getErrorKey()??> + ,"error-key":"${metrics.getErrorKey()}" + +} diff --git a/gravitee-reporter-elasticsearch/src/main/resources/freemarker/es7x/mapping/index-template-health.ftl b/gravitee-reporter-elasticsearch/src/main/resources/freemarker/es7x/mapping/index-template-health.ftl new file mode 100644 index 00000000..2a89122d --- /dev/null +++ b/gravitee-reporter-elasticsearch/src/main/resources/freemarker/es7x/mapping/index-template-health.ftl @@ -0,0 +1,44 @@ +<#ftl output_format="JSON"> +{ + "index_patterns": ["${indexName}-*"], + "settings": { + "index.number_of_shards":${numberOfShards}, + "index.number_of_replicas":${numberOfReplicas}, + "index.refresh_interval": "${refreshInterval}" + }, + "mappings": { + "properties": { + "api": { + "type": "keyword" + }, + "available": { + "type": "boolean", + "index": false + }, + "endpoint": { + "type": "keyword" + }, + "gateway": { + "type": "keyword" + }, + "response-time": { + "type": "integer" + }, + "state": { + "type": "integer", + "index": false + }, + "transition": { + "type": "boolean" + }, + "steps": { + "type": "object", + "enabled": false + }, + "success": { + "type": "boolean", + "index": false + } + } + } +} \ No newline at end of file diff --git a/gravitee-reporter-elasticsearch/src/main/resources/freemarker/es7x/mapping/index-template-log.ftl b/gravitee-reporter-elasticsearch/src/main/resources/freemarker/es7x/mapping/index-template-log.ftl new file mode 100644 index 00000000..09e07743 --- /dev/null +++ b/gravitee-reporter-elasticsearch/src/main/resources/freemarker/es7x/mapping/index-template-log.ftl @@ -0,0 +1,51 @@ +<#ftl output_format="JSON"> +{ + "index_patterns": ["${indexName}-*"], + "settings": { + "index.number_of_shards":${numberOfShards}, + "index.number_of_replicas":${numberOfReplicas}, + "index.refresh_interval": "${refreshInterval}" + }, + "mappings": { + "properties": { + "@timestamp": { + "type": "date" + }, + "api": { + "type": "keyword" + }, + "client-request": { + "type": "object", + "properties": { + "body":{ + "type": "keyword" + } + } + }, + "client-response": { + "type": "object", + "properties": { + "body":{ + "type": "keyword" + } + } + }, + "proxy-request": { + "type": "object", + "properties": { + "body":{ + "type": "keyword" + } + } + }, + "proxy-response": { + "type": "object", + "properties": { + "body":{ + "type": "keyword" + } + } + } + } + } +} \ No newline at end of file diff --git a/gravitee-reporter-elasticsearch/src/main/resources/freemarker/es7x/mapping/index-template-monitor.ftl b/gravitee-reporter-elasticsearch/src/main/resources/freemarker/es7x/mapping/index-template-monitor.ftl new file mode 100644 index 00000000..2c20d958 --- /dev/null +++ b/gravitee-reporter-elasticsearch/src/main/resources/freemarker/es7x/mapping/index-template-monitor.ftl @@ -0,0 +1,19 @@ +<#ftl output_format="JSON"> +{ + "index_patterns": ["${indexName}-*"], + "settings": { + "index.number_of_shards":${numberOfShards}, + "index.number_of_replicas":${numberOfReplicas}, + "index.refresh_interval": "${refreshInterval}" + }, + "mappings": { + "properties": { + "gateway": { + "type": "keyword" + }, + "hostname": { + "type": "keyword" + } + } + } +} \ No newline at end of file diff --git a/gravitee-reporter-elasticsearch/src/main/resources/freemarker/es7x/mapping/index-template-request.ftl b/gravitee-reporter-elasticsearch/src/main/resources/freemarker/es7x/mapping/index-template-request.ftl new file mode 100644 index 00000000..69c6aaa5 --- /dev/null +++ b/gravitee-reporter-elasticsearch/src/main/resources/freemarker/es7x/mapping/index-template-request.ftl @@ -0,0 +1,139 @@ +<#ftl output_format="JSON"> +{ + "index_patterns": ["${indexName}-*"], + "settings": { + "index.number_of_shards":${numberOfShards}, + "index.number_of_replicas":${numberOfReplicas}, + "index.refresh_interval": "${refreshInterval}" + }, + "mappings": { + "properties": { + "@timestamp": { + "type": "date" + }, + "api": { + "type": "keyword" + }, + "api-response-time": { + "type": "integer" + }, + "application": { + "type": "keyword" + }, + "endpoint": { + "type": "keyword" + }, + "gateway": { + "type": "keyword" + }, + "local-address": { + "type": "keyword", + "index": false + }, + "message": { + "type": "keyword", + "index": false + }, + "method": { + "type": "short" + }, + "plan": { + "type": "keyword" + }, + "proxy-latency": { + "type": "integer", + "index": false + }, + "remote-address": { + "type": "ip", + "index": false + }, + "geoip" : { + "properties": { + "continent_name":{ + "type": "keyword", + "index": true + }, + "country_iso_code":{ + "type": "keyword", + "index": true + }, + "region_name":{ + "type": "keyword", + "index": true + }, + "city_name":{ + "type": "keyword", + "index": true + }, + "location": { + "type": "geo_point" + } + } + }, + "request-content-length": { + "type": "integer", + "index": false + }, + "response-content-length": { + "type": "integer", + "index": false + }, + "response-time": { + "type": "integer" + }, + "status": { + "type": "short" + }, + "tenant": { + "type": "keyword" + }, + "transaction": { + "type": "keyword" + }, + "uri": { + "type": "keyword" + }, + "path": { + "type": "keyword" + }, + "mapped-path": { + "type": "keyword" + }, + "host": { + "type": "keyword" + }, + "user-agent": { + "type": "keyword" + }, + "user_agent": { + "properties": { + "name": { + "type": "keyword", + "index": true + }, + "os_name": { + "type": "keyword", + "index": true + } + } + }, + "user": { + "type": "keyword" + }, + "security-type": { + "type": "keyword", + "index": true + }, + "security-token": { + "type": "keyword", + "index": true + }, + "error-key": { + "type": "keyword", + "index": true + } + <#if extendedRequestMappingTemplate??>,<#include "/${extendedRequestMappingTemplate}"> + } + } +} \ No newline at end of file diff --git a/gravitee-repository-elasticsearch/src/main/java/io/gravitee/repository/elasticsearch/AbstractElasticsearchRepository.java b/gravitee-repository-elasticsearch/src/main/java/io/gravitee/repository/elasticsearch/AbstractElasticsearchRepository.java index 40be9737..64bc60d3 100644 --- a/gravitee-repository-elasticsearch/src/main/java/io/gravitee/repository/elasticsearch/AbstractElasticsearchRepository.java +++ b/gravitee-repository-elasticsearch/src/main/java/io/gravitee/repository/elasticsearch/AbstractElasticsearchRepository.java @@ -18,6 +18,7 @@ import io.gravitee.elasticsearch.client.Client; import io.gravitee.elasticsearch.index.IndexNameGenerator; import io.gravitee.elasticsearch.templating.freemarker.FreeMarkerComponent; +import io.gravitee.elasticsearch.version.ElasticsearchInfo; import org.springframework.beans.factory.annotation.Autowired; /** @@ -45,4 +46,7 @@ public abstract class AbstractElasticsearchRepository { */ @Autowired protected IndexNameGenerator indexNameGenerator; + + @Autowired + protected ElasticsearchInfo info; } diff --git a/gravitee-repository-elasticsearch/src/main/java/io/gravitee/repository/elasticsearch/analytics/query/AbstractElasticsearchQueryCommand.java b/gravitee-repository-elasticsearch/src/main/java/io/gravitee/repository/elasticsearch/analytics/query/AbstractElasticsearchQueryCommand.java index 28ac470b..2966e7fb 100644 --- a/gravitee-repository-elasticsearch/src/main/java/io/gravitee/repository/elasticsearch/analytics/query/AbstractElasticsearchQueryCommand.java +++ b/gravitee-repository-elasticsearch/src/main/java/io/gravitee/repository/elasticsearch/analytics/query/AbstractElasticsearchQueryCommand.java @@ -20,6 +20,7 @@ import io.gravitee.elasticsearch.model.SearchResponse; import io.gravitee.elasticsearch.templating.freemarker.FreeMarkerComponent; import io.gravitee.elasticsearch.utils.Type; +import io.gravitee.elasticsearch.version.ElasticsearchInfo; import io.gravitee.repository.analytics.query.AbstractQuery; import io.gravitee.repository.analytics.query.Query; import io.gravitee.repository.analytics.query.response.Response; @@ -73,6 +74,9 @@ public abstract class AbstractElasticsearchQueryCommand impl @Autowired protected RepositoryConfiguration configuration; + @Autowired + protected ElasticsearchInfo info; + private final static String TENANT_FIELD = "tenant"; /** @@ -122,12 +126,12 @@ Single execute(AbstractQuery query, Type type, String sQuery) result = this.client.search( this.indexNameGenerator.getIndexName(type, from, to, clusters), - type.getType(), + (info.getVersion().getMajorVersion() > 6) ? Type.DOC.getType() : type.getType(), sQuery); } else { result = this.client.search( this.indexNameGenerator.getTodayIndexName(type, clusters), - type.getType(), + (info.getVersion().getMajorVersion() > 6) ? Type.DOC.getType() : type.getType(), sQuery); } diff --git a/gravitee-repository-elasticsearch/src/main/java/io/gravitee/repository/elasticsearch/analytics/query/CountQueryCommand.java b/gravitee-repository-elasticsearch/src/main/java/io/gravitee/repository/elasticsearch/analytics/query/CountQueryCommand.java index c08af3df..778d54e1 100644 --- a/gravitee-repository-elasticsearch/src/main/java/io/gravitee/repository/elasticsearch/analytics/query/CountQueryCommand.java +++ b/gravitee-repository-elasticsearch/src/main/java/io/gravitee/repository/elasticsearch/analytics/query/CountQueryCommand.java @@ -54,7 +54,7 @@ public CountResponse executeQuery(Query query) throws AnalyticsEx private CountResponse toCountResponse(final SearchResponse response) { final CountResponse countResponse = new CountResponse(); - countResponse.setCount(response.getSearchHits().getTotal()); + countResponse.setCount(response.getSearchHits().getTotal().getValue()); return countResponse; } } diff --git a/gravitee-repository-elasticsearch/src/main/java/io/gravitee/repository/elasticsearch/healthcheck/ElasticsearchHealthCheckRepository.java b/gravitee-repository-elasticsearch/src/main/java/io/gravitee/repository/elasticsearch/healthcheck/ElasticsearchHealthCheckRepository.java index 48d76c24..289b0676 100644 --- a/gravitee-repository-elasticsearch/src/main/java/io/gravitee/repository/elasticsearch/healthcheck/ElasticsearchHealthCheckRepository.java +++ b/gravitee-repository-elasticsearch/src/main/java/io/gravitee/repository/elasticsearch/healthcheck/ElasticsearchHealthCheckRepository.java @@ -94,11 +94,11 @@ public ExtendedLog findById(String id) throws AnalyticsException { try { final Single result = this.client.search( this.indexNameGenerator.getWildcardIndexName(Type.HEALTH_CHECK), - Type.HEALTH_CHECK.getType(), + (info.getVersion().getMajorVersion() > 6) ? Type.DOC.getType() : Type.HEALTH_CHECK.getType(), sQuery); SearchResponse searchResponse = result.blockingGet(); - if (searchResponse.getSearchHits().getTotal() == 0) { + if (searchResponse.getSearchHits().getTotal().getValue() == 0) { throw new AnalyticsException("Health [" + id + "] does not exist"); } diff --git a/gravitee-repository-elasticsearch/src/main/java/io/gravitee/repository/elasticsearch/healthcheck/query/AbstractElasticsearchQueryCommand.java b/gravitee-repository-elasticsearch/src/main/java/io/gravitee/repository/elasticsearch/healthcheck/query/AbstractElasticsearchQueryCommand.java index f175bda4..5c1bd11e 100644 --- a/gravitee-repository-elasticsearch/src/main/java/io/gravitee/repository/elasticsearch/healthcheck/query/AbstractElasticsearchQueryCommand.java +++ b/gravitee-repository-elasticsearch/src/main/java/io/gravitee/repository/elasticsearch/healthcheck/query/AbstractElasticsearchQueryCommand.java @@ -18,6 +18,7 @@ import io.gravitee.elasticsearch.client.Client; import io.gravitee.elasticsearch.index.IndexNameGenerator; import io.gravitee.elasticsearch.templating.freemarker.FreeMarkerComponent; +import io.gravitee.elasticsearch.version.ElasticsearchInfo; import io.gravitee.repository.elasticsearch.healthcheck.ElasticsearchQueryCommand; import io.gravitee.repository.healthcheck.query.Query; import io.gravitee.repository.healthcheck.query.Response; @@ -62,6 +63,9 @@ public abstract class AbstractElasticsearchQueryCommand impl @Autowired protected IndexNameGenerator indexNameGenerator; + @Autowired + protected ElasticsearchInfo info; + /** * Create the elasticsearch query * @param templateName Freemarker template name diff --git a/gravitee-repository-elasticsearch/src/main/java/io/gravitee/repository/elasticsearch/healthcheck/query/AverageAvailabilityCommand.java b/gravitee-repository-elasticsearch/src/main/java/io/gravitee/repository/elasticsearch/healthcheck/query/AverageAvailabilityCommand.java index a1194add..93957bd8 100644 --- a/gravitee-repository-elasticsearch/src/main/java/io/gravitee/repository/elasticsearch/healthcheck/query/AverageAvailabilityCommand.java +++ b/gravitee-repository-elasticsearch/src/main/java/io/gravitee/repository/elasticsearch/healthcheck/query/AverageAvailabilityCommand.java @@ -73,7 +73,7 @@ public AvailabilityResponse executeQuery(Query query) thro final Single result = this.client.search( this.indexNameGenerator.getIndexName(Type.HEALTH_CHECK, from, now), - Type.HEALTH_CHECK.getType(), + (info.getVersion().getMajorVersion() > 6) ? Type.DOC.getType() : Type.HEALTH_CHECK.getType(), sQuery); return this.toAvailabilityResponseResponse(result.blockingGet()); diff --git a/gravitee-repository-elasticsearch/src/main/java/io/gravitee/repository/elasticsearch/healthcheck/query/AverageDateHistogramCommand.java b/gravitee-repository-elasticsearch/src/main/java/io/gravitee/repository/elasticsearch/healthcheck/query/AverageDateHistogramCommand.java index 70318634..1fe53555 100644 --- a/gravitee-repository-elasticsearch/src/main/java/io/gravitee/repository/elasticsearch/healthcheck/query/AverageDateHistogramCommand.java +++ b/gravitee-repository-elasticsearch/src/main/java/io/gravitee/repository/elasticsearch/healthcheck/query/AverageDateHistogramCommand.java @@ -84,7 +84,7 @@ public DateHistogramResponse executeQuery(Query query) th final String sQuery = this.createQuery(TEMPLATE, dateHistogramQuery, roundedFrom, roundedTo); final Single result = this.client.search( this.indexNameGenerator.getIndexName(Type.HEALTH_CHECK, from, to), - Type.HEALTH_CHECK.getType(), + (info.getVersion().getMajorVersion() > 6) ? Type.DOC.getType() : Type.HEALTH_CHECK.getType(), sQuery); return this.toAvailabilityResponseResponse(result.blockingGet(), dateHistogramQuery); } catch (Exception eex) { diff --git a/gravitee-repository-elasticsearch/src/main/java/io/gravitee/repository/elasticsearch/healthcheck/query/AverageResponseTimeCommand.java b/gravitee-repository-elasticsearch/src/main/java/io/gravitee/repository/elasticsearch/healthcheck/query/AverageResponseTimeCommand.java index 762e18e2..785642bb 100644 --- a/gravitee-repository-elasticsearch/src/main/java/io/gravitee/repository/elasticsearch/healthcheck/query/AverageResponseTimeCommand.java +++ b/gravitee-repository-elasticsearch/src/main/java/io/gravitee/repository/elasticsearch/healthcheck/query/AverageResponseTimeCommand.java @@ -73,7 +73,7 @@ public AverageResponseTimeResponse executeQuery(Query result = this.client.search( this.indexNameGenerator.getIndexName(Type.HEALTH_CHECK, from, now), - Type.HEALTH_CHECK.getType(), + (info.getVersion().getMajorVersion() > 6) ? Type.DOC.getType() : Type.HEALTH_CHECK.getType(), sQuery); return this.toAverageResponseTimeResponse(result.blockingGet()); } catch (Exception eex) { diff --git a/gravitee-repository-elasticsearch/src/main/java/io/gravitee/repository/elasticsearch/healthcheck/query/LogsCommand.java b/gravitee-repository-elasticsearch/src/main/java/io/gravitee/repository/elasticsearch/healthcheck/query/LogsCommand.java index 87394c23..3920be7a 100644 --- a/gravitee-repository-elasticsearch/src/main/java/io/gravitee/repository/elasticsearch/healthcheck/query/LogsCommand.java +++ b/gravitee-repository-elasticsearch/src/main/java/io/gravitee/repository/elasticsearch/healthcheck/query/LogsCommand.java @@ -75,7 +75,7 @@ public LogsResponse executeQuery(Query query) throws AnalyticsExce final Single result = this.client.search( this.indexNameGenerator.getIndexName(Type.HEALTH_CHECK, from, to), - Type.HEALTH_CHECK.getType(), + (info.getVersion().getMajorVersion() > 6) ? Type.DOC.getType() : Type.HEALTH_CHECK.getType(), sQuery); return this.toLogsResponse(result.blockingGet()); } catch (ElasticsearchException eex) { @@ -86,7 +86,7 @@ public LogsResponse executeQuery(Query query) throws AnalyticsExce private LogsResponse toLogsResponse(final SearchResponse response) { SearchHits hits = response.getSearchHits(); - LogsResponse logsResponse = new LogsResponse(hits.getTotal()); + LogsResponse logsResponse = new LogsResponse(hits.getTotal().getValue()); List logs = new ArrayList<>(hits.getHits().size()); for (SearchHit hit : hits.getHits()) { diff --git a/gravitee-repository-elasticsearch/src/main/java/io/gravitee/repository/elasticsearch/log/ElasticLogRepository.java b/gravitee-repository-elasticsearch/src/main/java/io/gravitee/repository/elasticsearch/log/ElasticLogRepository.java index 9708cdb8..030ca2d1 100644 --- a/gravitee-repository-elasticsearch/src/main/java/io/gravitee/repository/elasticsearch/log/ElasticLogRepository.java +++ b/gravitee-repository-elasticsearch/src/main/java/io/gravitee/repository/elasticsearch/log/ElasticLogRepository.java @@ -80,7 +80,7 @@ public TabularResponse query(final TabularQuery query) throws AnalyticsException if (isEmpty(logQueryString)) { final Single result = this.client.search( this.indexNameGenerator.getIndexName(Type.REQUEST, from, to), - Type.REQUEST.getType(), + (info.getVersion().getMajorVersion() > 6) ? Type.DOC.getType() : Type.REQUEST.getType(), this.createElasticsearchJsonQuery(query)); return this.toTabularResponse(result.blockingGet()); @@ -88,7 +88,7 @@ public TabularResponse query(final TabularQuery query) throws AnalyticsException final String sQuery = this.createElasticsearchJsonQuery(tabularQueryBuilder.query(logQueryString).build()); Single result = this.client.search( this.indexNameGenerator.getIndexName(Type.LOG, from, to), - Type.LOG.getType(), + (info.getVersion().getMajorVersion() > 6) ? Type.DOC.getType() : Type.LOG.getType(), sQuery); final SearchResponse searchResponse = result.blockingGet(); @@ -106,7 +106,7 @@ public TabularResponse query(final TabularQuery query) throws AnalyticsException } result = this.client.search( this.indexNameGenerator.getIndexName(Type.REQUEST, from, to), - Type.REQUEST.getType(), + (info.getVersion().getMajorVersion() > 6) ? Type.DOC.getType() : Type.REQUEST.getType(), this.createElasticsearchJsonQuery(tqb.build())); } @@ -163,11 +163,11 @@ public ExtendedLog findById(final String requestId, final Long timestamp) throws try { Single result = this.client.search( (timestamp == null) ? this.indexNameGenerator.getWildcardIndexName(Type.REQUEST) : this.indexNameGenerator.getIndexName(Type.REQUEST, Instant.ofEpochMilli(timestamp)), - Type.REQUEST.getType(), + (info.getVersion().getMajorVersion() > 6) ? Type.DOC.getType() : Type.REQUEST.getType(), sQuery); SearchResponse searchResponse = result.blockingGet(); - if (searchResponse.getSearchHits().getTotal() == 0) { + if (searchResponse.getSearchHits().getTotal().getValue() == 0) { throw new AnalyticsException("Request [" + requestId + "] does not exist"); } @@ -179,11 +179,11 @@ public ExtendedLog findById(final String requestId, final Long timestamp) throws // Search index must be updated in case of per-type index searchHitIndex = searchHitIndex.replaceAll(Type.REQUEST.getType(), Type.LOG.getType()); - result = this.client.search(searchHitIndex, Type.LOG.getType(), sQuery); + result = this.client.search(searchHitIndex, (info.getVersion().getMajorVersion() > 6) ? Type.DOC.getType() : Type.LOG.getType(), sQuery); searchResponse = result.blockingGet(); JsonNode log = null; - if (searchResponse.getSearchHits().getTotal() != 0) { + if (searchResponse.getSearchHits().getTotal().getValue() != 0) { log = searchResponse.getSearchHits().getHits().get(0).getSource(); } @@ -196,7 +196,7 @@ public ExtendedLog findById(final String requestId, final Long timestamp) throws private TabularResponse toTabularResponse(final SearchResponse response) { final SearchHits hits = response.getSearchHits(); - final TabularResponse tabularResponse = new TabularResponse(hits.getTotal()); + final TabularResponse tabularResponse = new TabularResponse(hits.getTotal().getValue()); final List logs = new ArrayList<>(hits.getHits().size()); for (int i = 0; i < hits.getHits().size(); i++) { logs.add(LogBuilder.createLog(hits.getHits().get(i))); diff --git a/gravitee-repository-elasticsearch/src/main/java/io/gravitee/repository/elasticsearch/monitoring/ElasticsearchMonitoringRepository.java b/gravitee-repository-elasticsearch/src/main/java/io/gravitee/repository/elasticsearch/monitoring/ElasticsearchMonitoringRepository.java index 313e8da8..52fe2130 100644 --- a/gravitee-repository-elasticsearch/src/main/java/io/gravitee/repository/elasticsearch/monitoring/ElasticsearchMonitoringRepository.java +++ b/gravitee-repository-elasticsearch/src/main/java/io/gravitee/repository/elasticsearch/monitoring/ElasticsearchMonitoringRepository.java @@ -67,7 +67,7 @@ public MonitoringResponse query(final String gatewayId) { try { final Single result = this.client.search( this.indexNameGenerator.getTodayIndexName(Type.MONITOR), - Type.MONITOR.getType(), + (info.getVersion().getMajorVersion() > 6) ? Type.DOC.getType() : Type.MONITOR.getType(), sQuery); final SearchHits hits = result.blockingGet().getSearchHits(); diff --git a/gravitee-repository-elasticsearch/src/main/java/io/gravitee/repository/elasticsearch/spring/ElasticsearchRepositoryConfiguration.java b/gravitee-repository-elasticsearch/src/main/java/io/gravitee/repository/elasticsearch/spring/ElasticsearchRepositoryConfiguration.java index 8bf1b99e..5cab554a 100644 --- a/gravitee-repository-elasticsearch/src/main/java/io/gravitee/repository/elasticsearch/spring/ElasticsearchRepositoryConfiguration.java +++ b/gravitee-repository-elasticsearch/src/main/java/io/gravitee/repository/elasticsearch/spring/ElasticsearchRepositoryConfiguration.java @@ -22,6 +22,7 @@ import io.gravitee.elasticsearch.index.MultiTypeIndexNameGenerator; import io.gravitee.elasticsearch.index.PerTypeIndexNameGenerator; import io.gravitee.elasticsearch.templating.freemarker.FreeMarkerComponent; +import io.gravitee.elasticsearch.version.ElasticsearchInfo; import io.gravitee.repository.elasticsearch.analytics.spring.AnalyticsConfiguration; import io.gravitee.repository.elasticsearch.configuration.RepositoryConfiguration; import io.gravitee.repository.elasticsearch.healthcheck.spring.HealthCheckConfiguration; @@ -82,20 +83,23 @@ public Client client(RepositoryConfiguration repositoryConfiguration) { } @Bean - public IndexNameGenerator indexNameGenerator(RepositoryConfiguration repositoryConfiguration, Client client) { + public IndexNameGenerator indexNameGenerator(RepositoryConfiguration repositoryConfiguration, ElasticsearchInfo info) { + if (info.getVersion().getMajorVersion() == 6 || info.getVersion().getMajorVersion() == 7 || repositoryConfiguration.isPerTypeIndex()) { + return new PerTypeIndexNameGenerator(repositoryConfiguration.getIndexName()); + } else { + return new MultiTypeIndexNameGenerator(repositoryConfiguration.getIndexName()); + } + } + + @Bean + public ElasticsearchInfo elasticsearchInfo(Client client) { // Wait for a connection to ES and retry each 5 seconds - Single singleVersion = client.getVersion() + Single singleVersion = client.getInfo() .retryWhen(error -> error.flatMap( throwable -> Observable.just(new Object()).delay(5, TimeUnit.SECONDS).toFlowable(BackpressureStrategy.LATEST))); singleVersion.subscribe(); - Integer version = singleVersion.blockingGet(); - - if (version == 6 || repositoryConfiguration.isPerTypeIndex()) { - return new PerTypeIndexNameGenerator(repositoryConfiguration.getIndexName()); - } else { - return new MultiTypeIndexNameGenerator(repositoryConfiguration.getIndexName()); - } + return singleVersion.blockingGet(); } } diff --git a/gravitee-repository-elasticsearch/src/main/resources/freemarker/groupBy.ftl b/gravitee-repository-elasticsearch/src/main/resources/freemarker/groupBy.ftl index e5596887..bbf6a6b9 100644 --- a/gravitee-repository-elasticsearch/src/main/resources/freemarker/groupBy.ftl +++ b/gravitee-repository-elasticsearch/src/main/resources/freemarker/groupBy.ftl @@ -49,14 +49,13 @@ <#sep>, ]} - } <#else> "by_${query.field()}": { "terms":{ "field":"${query.field()}", "size": 1000 <#if query.sort()?has_content> - ,"order":{ + ,"order": { "${query.sort().getType().name()?lower_case}_${query.sort().getField()}":"${query.sort().getOrder()?lower_case}" } }, @@ -72,7 +71,6 @@ <#default> <#break> - } } diff --git a/pom.xml b/pom.xml index 43fb890c..bff8cdce 100644 --- a/pom.xml +++ b/pom.xml @@ -212,6 +212,17 @@ + + es7x + + test + + + + 7x + + + @@ -313,6 +324,47 @@ test + + org.apache.logging.log4j + log4j-core + 2.9.1 + test + + + + + es-7x + + + elasticsearch + 7x + + + + 7.3.0 + + + + org.elasticsearch + elasticsearch + ${elasticsearch.version} + test + + + + org.elasticsearch.client + transport + ${elasticsearch.version} + test + + + + io.netty + netty-all + 4.1.36.Final + test + + org.apache.logging.log4j log4j-core