Skip to content

Commit

Permalink
Merge.1.to.3 (#87)
Browse files Browse the repository at this point in the history 
* chore(): Prepare next version

* chore(): Prepare next version

* fix(template): index remote address

fix gravitee-io/issues#2895

* release(1.25.13)

* chore(): Prepare next version

* fix(elasticsearch): Update field type for body payload

Closes gravitee-io/issues#2952

* fix(httpClient): allow path in endpoints

fix gravitee-io/issues#3030

* fix(groupBy): aggregation by response-time doesnt work

fix gravitee-io/issues#3041

* release(1.30.1)

* chore(): Prepare next version

* fix: analytic widgets with latency displayed hits instead of latency

Closes gravitee-io/issues#3041

* release(1.30.2)

* chore(): Prepare next version

* fix: cannot index a cpu loadaverage as long

Closes gravitee-io/issues#2692

* release(1.25.14)

* release(1.30.3)

* chore(): Prepare next version

* fix(reporter): Do not stop if bulk indexer encounters error

Closes gravitee-io/issues#3630

* chore(): upgrade parent to fix gpg error

* release(1.30.4)

Co-authored-by: Gravitee.io Bot <contact@gravitee.io>
Co-authored-by: David BRASSELY <brasseld@gmail.com>
Co-authored-by: Azize Elamrani <azize.elamrani@gmail.com>
  • Loading branch information
@NicolasGeraud @graviteeio
@brasseld @aelamrani
4 people committed May 6, 2020
1 parent 4414987 commit 957a321
Show file tree
Hide file tree
Showing 15 changed files with 140 additions and 65 deletions.
52 changes: 32 additions & 20 deletions ...-common-elasticsearch/src/main/java/io/gravitee/elasticsearch/client/http/HttpClient.java
View file
Expand Up @@ -51,6 +51,7 @@

/**
* @author David BRASSELY (david.brassely at graviteesource.com)
* @author Nicolas GERAUD (nicolas.geraud at graviteesource.com)
* @author GraviteeSource Team
*/
public class HttpClient implements Client {
Expand All @@ -63,13 +64,13 @@ public class HttpClient implements Client {
private static final String HTTPS_SCHEME = "https";
private static final String CONTENT_TYPE = MediaType.APPLICATION_JSON + ";charset=UTF-8";

private static final String URL_ROOT = "/";
private static final String URL_STATE_CLUSTER = "/_cluster/health";
private static final String URL_BULK = "/_bulk";
private static final String URL_TEMPLATE = "/_template";
private static final String URL_INGEST = "/_ingest/pipeline";
private static final String URL_SEARCH = "/_search?ignore_unavailable=true";
private static final String URL_COUNT = "/_count?ignore_unavailable=true";
private static String URL_ROOT;
private static String URL_STATE_CLUSTER;
private static String URL_BULK;
private static String URL_TEMPLATE;
private static String URL_INGEST;
private static String URL_SEARCH;
private static String URL_COUNT;

@Autowired
private Vertx vertx;
Expand Down Expand Up @@ -104,6 +105,7 @@ public void initialize() {
if (! configuration.getEndpoints().isEmpty()) {
final Endpoint endpoint = configuration.getEndpoints().get(0);
final URI elasticEdpt = URI.create(endpoint.getUrl());
initializePaths(elasticEdpt);

WebClientOptions options = new WebClientOptions()
.setDefaultHost(elasticEdpt.getHost())
Expand All @@ -128,25 +130,35 @@ public void initialize() {
this.configuration.getPassword());
}

((WebClientInternal) this.httpClient.getDelegate()).addInterceptor(new Handler<HttpContext<?>>() {
@Override
public void handle(HttpContext context) {
context.request()
.timeout(configuration.getRequestTimeout())
.putHeader(HttpHeaders.ACCEPT, CONTENT_TYPE)
.putHeader(HttpHeaders.ACCEPT_CHARSET, StandardCharsets.UTF_8.name());

// Basic authentication
if (authorizationHeader != null) {
context.request().putHeader(HttpHeaders.AUTHORIZATION, authorizationHeader);
}
((WebClientInternal) this.httpClient.getDelegate()).addInterceptor(context -> {
context.request()
.timeout(configuration.getRequestTimeout())
.putHeader(HttpHeaders.ACCEPT, CONTENT_TYPE)
.putHeader(HttpHeaders.ACCEPT_CHARSET, StandardCharsets.UTF_8.name());

context.next();
// Basic authentication
if (authorizationHeader != null) {
context.request().putHeader(HttpHeaders.AUTHORIZATION, authorizationHeader);
}

context.next();
});

}
}

private void initializePaths(URI uri) {
String urlPrefix = uri.getPath().replaceAll("/$", "");

URL_ROOT = urlPrefix + "/";
URL_STATE_CLUSTER = urlPrefix + "/_cluster/health";
URL_BULK = urlPrefix + "/_bulk";
URL_TEMPLATE = urlPrefix + "/_template";
URL_INGEST = urlPrefix + "/_ingest/pipeline";
URL_SEARCH = urlPrefix + "/_search?ignore_unavailable=true";
URL_COUNT = urlPrefix + "/_count?ignore_unavailable=true";
}

@Override
public Single<ElasticsearchInfo> getInfo() throws ElasticsearchException {
return httpClient
Expand Down
9 changes: 8 additions & 1 deletion ...r-elasticsearch/src/main/java/io/gravitee/reporter/elasticsearch/indexer/BulkIndexer.java
View file
Expand Up @@ -18,9 +18,13 @@
import io.gravitee.elasticsearch.client.Client;
import io.gravitee.reporter.api.Reportable;
import io.gravitee.reporter.elasticsearch.config.ReporterConfiguration;
import io.reactivex.Flowable;
import io.reactivex.functions.Function;
import io.reactivex.processors.PublishProcessor;
import io.reactivex.schedulers.Schedulers;
import io.vertx.core.buffer.Buffer;
import io.vertx.reactivex.core.Vertx;
import org.reactivestreams.Publisher;
import org.springframework.beans.factory.annotation.Autowired;

import javax.annotation.PostConstruct;
Expand Down Expand Up @@ -55,7 +59,10 @@ public void initialize() {
bulkProcessor
.onBackpressureBuffer()
.observeOn(Schedulers.io())
.map(this::transform)
.flatMap((Function<Reportable, Publisher<Buffer>>) reportable ->
Flowable.just(reportable)
.map(this::transform)
.onErrorResumeNext(Flowable.empty()))
.buffer(
configuration.getFlushInterval(),
TimeUnit.SECONDS,
Expand Down
8 changes: 4 additions & 4 deletions ...-reporter-elasticsearch/src/main/resources/freemarker/es5x/mapping/index-template-log.ftl
View file
Expand Up @@ -19,31 +19,31 @@
"type": "object",
"properties": {
"body":{
"type": "keyword"
"type": "text"
}
}
},
"client-response": {
"type": "object",
"properties": {
"body":{
"type": "keyword"
"type": "text"
}
}
},
"proxy-request": {
"type": "object",
"properties": {
"body":{
"type": "keyword"
"type": "text"
}
}
},
"proxy-response": {
"type": "object",
"properties": {
"body":{
"type": "keyword"
"type": "text"
}
}
}
Expand Down
23 changes: 22 additions & 1 deletion ...orter-elasticsearch/src/main/resources/freemarker/es5x/mapping/index-template-monitor.ftl
View file
Expand Up @@ -9,6 +9,27 @@
"mappings": {
"monitor": {
"properties": {
"os": {
"properties": {
"cpu": {
"properties": {
"load_average": {
"properties": {
"1m": {
"type": "float"
},
"5m": {
"type": "float"
},
"15m": {
"type": "float"
}
}
}
}
}
}
},
"gateway": {
"type": "keyword"
},
Expand All @@ -18,4 +39,4 @@
}
}
}
}
}
3 changes: 1 addition & 2 deletions ...orter-elasticsearch/src/main/resources/freemarker/es5x/mapping/index-template-request.ftl
View file
Expand Up @@ -45,8 +45,7 @@
"index": false
},
"remote-address": {
"type": "ip",
"index": false
"type": "ip"
},
"geoip" : {
"properties": {
Expand Down
8 changes: 4 additions & 4 deletions ...-reporter-elasticsearch/src/main/resources/freemarker/es6x/mapping/index-template-log.ftl
View file
Expand Up @@ -19,31 +19,31 @@
"type": "object",
"properties": {
"body":{
"type": "keyword"
"type": "text"
}
}
},
"client-response": {
"type": "object",
"properties": {
"body":{
"type": "keyword"
"type": "text"
}
}
},
"proxy-request": {
"type": "object",
"properties": {
"body":{
"type": "keyword"
"type": "text"
}
}
},
"proxy-response": {
"type": "object",
"properties": {
"body":{
"type": "keyword"
"type": "text"
}
}
}
Expand Down
23 changes: 22 additions & 1 deletion ...orter-elasticsearch/src/main/resources/freemarker/es6x/mapping/index-template-monitor.ftl
View file
Expand Up @@ -9,6 +9,27 @@
"mappings": {
"monitor": {
"properties": {
"os": {
"properties": {
"cpu": {
"properties": {
"load_average": {
"properties": {
"1m": {
"type": "float"
},
"5m": {
"type": "float"
},
"15m": {
"type": "float"
}
}
}
}
}
}
},
"gateway": {
"type": "keyword"
},
Expand All @@ -18,4 +39,4 @@
}
}
}
}
}
3 changes: 1 addition & 2 deletions ...orter-elasticsearch/src/main/resources/freemarker/es6x/mapping/index-template-request.ftl
View file
Expand Up @@ -45,8 +45,7 @@
"index": false
},
"remote-address": {
"type": "ip",
"index": false
"type": "ip"
},
"geoip" : {
"properties": {
Expand Down
8 changes: 4 additions & 4 deletions ...-reporter-elasticsearch/src/main/resources/freemarker/es7x/mapping/index-template-log.ftl
View file
Expand Up @@ -18,31 +18,31 @@
"type": "object",
"properties": {
"body":{
"type": "keyword"
"type": "text"
}
}
},
"client-response": {
"type": "object",
"properties": {
"body":{
"type": "keyword"
"type": "text"
}
}
},
"proxy-request": {
"type": "object",
"properties": {
"body":{
"type": "keyword"
"type": "text"
}
}
},
"proxy-response": {
"type": "object",
"properties": {
"body":{
"type": "keyword"
"type": "text"
}
}
}
Expand Down
23 changes: 22 additions & 1 deletion ...orter-elasticsearch/src/main/resources/freemarker/es7x/mapping/index-template-monitor.ftl
View file
Expand Up @@ -8,6 +8,27 @@
},
"mappings": {
"properties": {
"os": {
"properties": {
"cpu": {
"properties": {
"load_average": {
"properties": {
"1m": {
"type": "float"
},
"5m": {
"type": "float"
},
"15m": {
"type": "float"
}
}
}
}
}
}
},
"gateway": {
"type": "keyword"
},
Expand All @@ -16,4 +37,4 @@
}
}
}
}
}
3 changes: 1 addition & 2 deletions ...orter-elasticsearch/src/main/resources/freemarker/es7x/mapping/index-template-request.ftl
View file
Expand Up @@ -44,8 +44,7 @@
"index": false
},
"remote-address": {
"type": "ip",
"index": false
"type": "ip"
},
"geoip" : {
"properties": {
Expand Down

0 comments on commit 957a321

Please sign in to comment.