diff --git a/pom.xml b/pom.xml index 02c3d968db85..9081d0ce20c3 100644 --- a/pom.xml +++ b/pom.xml @@ -75,7 +75,7 @@ 1.1.7 1.7.21 8.5.16 - 2.4.4 + 5.0.0 3.15.0.1090 3.7.0 2.6.6 @@ -842,6 +842,11 @@ log4j-over-slf4j ${slf4j.version} + + org.apache.logging.log4j + log4j-api + 2.7 + ch.qos.logback logback-access diff --git a/server/sonar-search/pom.xml b/server/sonar-search/pom.xml index 96adff700111..4f56354b14c7 100644 --- a/server/sonar-search/pom.xml +++ b/server/sonar-search/pom.xml @@ -35,16 +35,14 @@ - org.slf4j - slf4j-api + org.apache.logging.log4j + log4j-api + 2.6.2 - ch.qos.logback - logback-classic - - - ch.qos.logback - logback-core + org.apache.logging.log4j + log4j-core + 2.6.2 diff --git a/server/sonar-search/src/main/java/org/sonar/search/SearchServer.java b/server/sonar-search/src/main/java/org/sonar/search/SearchServer.java index 21e75193376a..bdcce3718ab2 100644 --- a/server/sonar-search/src/main/java/org/sonar/search/SearchServer.java +++ b/server/sonar-search/src/main/java/org/sonar/search/SearchServer.java @@ -19,12 +19,13 @@ */ package org.sonar.search; +import java.io.IOException; import org.apache.lucene.util.StringHelper; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.node.Node; -import org.elasticsearch.node.NodeBuilder; +import org.elasticsearch.node.NodeValidationException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sonar.process.Jmx; @@ -56,11 +57,14 @@ public void start() { if (esSettings.getAsInt(MIMINUM_MASTER_NODES, 1) >= 2) { LOGGER.info("Elasticsearch is waiting {} for {} node(s) to be up to start.", esSettings.get(INITIAL_STATE_TIMEOUT), - esSettings.get(MIMINUM_MASTER_NODES) - ); + esSettings.get(MIMINUM_MASTER_NODES)); + } + node = new Node(settings.build()); + try { + node.start(); + } catch (NodeValidationException e) { + throw new RuntimeException("Failed to start ES", e); } - node = NodeBuilder.nodeBuilder().settings(esSettings).build(); - node.start(); } // copied from https://github.com/elastic/elasticsearch/blob/v2.3.3/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java @@ -72,10 +76,10 @@ private static void initBootstrap() { @Override public Status getStatus() { boolean esStatus = node != null && node.client().admin().cluster().prepareHealth() - .setWaitForYellowStatus() - .setTimeout(TimeValue.timeValueSeconds(30L)) - .get() - .getStatus() != ClusterHealthStatus.RED; + .setWaitForYellowStatus() + .setTimeout(TimeValue.timeValueSeconds(30L)) + .get() + .getStatus() != ClusterHealthStatus.RED; if (esStatus) { return Status.OPERATIONAL; } @@ -97,7 +101,11 @@ public void awaitStop() { @Override public void stop() { if (node != null && !node.isClosed()) { - node.close(); + try { + node.close(); + } catch (IOException e) { + LOGGER.error("Failed to stop ES cleanly", e); + } } Jmx.unregister(EsSettingsMBean.OBJECT_NAME); } diff --git a/server/sonar-search/src/test/java/org/sonar/search/SearchServerTest.java b/server/sonar-search/src/test/java/org/sonar/search/SearchServerTest.java index 407a99ecc271..a431475b7416 100644 --- a/server/sonar-search/src/test/java/org/sonar/search/SearchServerTest.java +++ b/server/sonar-search/src/test/java/org/sonar/search/SearchServerTest.java @@ -21,6 +21,8 @@ import java.io.IOException; import java.net.InetAddress; +import java.util.Collection; +import java.util.Collections; import java.util.Properties; import org.elasticsearch.client.Client; import org.elasticsearch.client.transport.NoNodeAvailableException; @@ -28,6 +30,7 @@ import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.InetSocketTransportAddress; +import org.elasticsearch.plugins.Plugin; import org.junit.After; import org.junit.Rule; import org.junit.Test; @@ -86,6 +89,7 @@ public void log_information_on_startup() throws IOException { Logger logger = mock(Logger.class); underTest.LOGGER = logger; underTest.start(); + System.out.println(org.apache.logging.log4j.Logger.class); verify(logger).info(eq("Elasticsearch is waiting {} for {} node(s) to be up to start."), eq("1s"), eq("2")); } @@ -109,8 +113,10 @@ public void start_stop_server() throws Exception { assertThat(underTest.getStatus()).isEqualTo(Monitored.Status.OPERATIONAL); Settings settings = Settings.builder().put("cluster.name", A_CLUSTER_NAME).build(); - client = TransportClient.builder().settings(settings).build() - .addTransportAddress(new InetSocketTransportAddress(InetAddress.getLoopbackAddress(), port)); + Collection> plugins = Collections.emptyList(); + client = new TransportClient(settings, plugins) { + + }.addTransportAddress(new InetSocketTransportAddress(InetAddress.getLoopbackAddress(), port)); assertThat(client.admin().cluster().prepareClusterStats().get().getStatus()).isEqualTo(ClusterHealthStatus.GREEN); underTest.stop(); diff --git a/server/sonar-server/pom.xml b/server/sonar-server/pom.xml index 1d73103fa0b2..59edc4ed98d2 100644 --- a/server/sonar-server/pom.xml +++ b/server/sonar-server/pom.xml @@ -27,6 +27,10 @@ org.slf4j jul-to-slf4j + + org.apache.logging.log4j + log4j-api + ch.qos.logback logback-access @@ -148,11 +152,10 @@ elasticsearch - org.elasticsearch - elasticsearch + org.elasticsearch.test + framework ${elasticsearch.version} test - test-jar net.java.dev.jna diff --git a/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndex.java b/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndex.java index f0babc80127a..263fd4782448 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndex.java @@ -33,10 +33,10 @@ import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.bucket.filters.FiltersAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.filters.InternalFilters; -import org.elasticsearch.search.aggregations.bucket.filters.InternalFilters.Bucket; +import org.elasticsearch.search.aggregations.bucket.filters.InternalFilters.InternalBucket; import org.elasticsearch.search.aggregations.metrics.tophits.InternalTopHits; -import org.elasticsearch.search.aggregations.metrics.tophits.TopHitsBuilder; -import org.elasticsearch.search.highlight.HighlightBuilder; +import org.elasticsearch.search.aggregations.metrics.tophits.TopHitsAggregationBuilder; +import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.ScoreSortBuilder; import org.sonar.server.es.EsClient; @@ -67,7 +67,7 @@ public ComponentIndex(EsClient client, AuthorizationTypeSupport authorizationTyp this.authorizationTypeSupport = authorizationTypeSupport; } - private static HighlightBuilder.Field createHighlighter() { + private static HighlightBuilder.Field createHighlighterField() { HighlightBuilder.Field field = new HighlightBuilder.Field(FIELD_NAME); field.highlighterType("fvh"); field.matchedFields( @@ -105,23 +105,25 @@ ComponentIndexResults search(ComponentIndexQuery query, ComponentTextSearchFeatu } private static FiltersAggregationBuilder createAggregation(ComponentIndexQuery query) { - FiltersAggregationBuilder filtersAggregation = AggregationBuilders.filters(FILTERS_AGGREGATION_NAME) + return AggregationBuilders.filters( + FILTERS_AGGREGATION_NAME, + query.getQualifiers().stream().map(q -> termQuery(FIELD_QUALIFIER, q)).toArray(QueryBuilder[]::new)) .subAggregation(createSubAggregation(query)); - query.getQualifiers().forEach(q -> filtersAggregation.filter(q, termQuery(FIELD_QUALIFIER, q))); - return filtersAggregation; } - private static TopHitsBuilder createSubAggregation(ComponentIndexQuery query) { - TopHitsBuilder sub = AggregationBuilders.topHits(DOCS_AGGREGATION_NAME) - .setHighlighterEncoder("html") - .setHighlighterPreTags("") - .setHighlighterPostTags("") - .addHighlightedField(createHighlighter()) - .setFrom(query.getSkip()) - .setSize(query.getLimit()) - .addSort(new ScoreSortBuilder()) - .addSort(new FieldSortBuilder(ComponentIndexDefinition.FIELD_NAME)); - return sub.setFetchSource(false); + private static TopHitsAggregationBuilder createSubAggregation(ComponentIndexQuery query) { + return AggregationBuilders.topHits(DOCS_AGGREGATION_NAME) + .highlighter(new HighlightBuilder() + .encoder("html") + .preTags("") + .postTags("") + .field(createHighlighterField()) + ) + .from(query.getSkip()) + .size(query.getLimit()) + .sort(new ScoreSortBuilder()) + .sort(new FieldSortBuilder(ComponentIndexDefinition.FIELD_NAME)) + .fetchSource(false); } private QueryBuilder createQuery(ComponentIndexQuery query, ComponentTextSearchFeature... features) { @@ -139,14 +141,14 @@ private QueryBuilder createQuery(ComponentIndexQuery query, ComponentTextSearchF private static ComponentIndexResults aggregationsToQualifiers(SearchResponse response) { InternalFilters filtersAgg = response.getAggregations().get(FILTERS_AGGREGATION_NAME); - List buckets = filtersAgg.getBuckets(); + List buckets = filtersAgg.getBuckets(); return ComponentIndexResults.newBuilder() .setQualifiers( buckets.stream().map(ComponentIndex::bucketToQualifier)) .build(); } - private static ComponentHitsPerQualifier bucketToQualifier(Bucket bucket) { + private static ComponentHitsPerQualifier bucketToQualifier(InternalBucket bucket) { InternalTopHits docs = bucket.getAggregations().get(DOCS_AGGREGATION_NAME); SearchHits hitList = docs.getHits(); diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/DefaultIndexSettings.java b/server/sonar-server/src/main/java/org/sonar/server/es/DefaultIndexSettings.java index 7937f140eac2..24ca6110d783 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/DefaultIndexSettings.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/DefaultIndexSettings.java @@ -20,6 +20,7 @@ package org.sonar.server.es; import java.util.Arrays; +import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; @@ -43,18 +44,17 @@ public class DefaultIndexSettings { public static final String TYPE = "type"; public static final String INDEX = "index"; - public static final String INDEX_SEARCHABLE_FOR_TEXT = "analyzed"; // ES 5: remove this constant and replace usages with INDEX_SEARCHABLE - public static final String INDEX_SEARCHABLE_FOR_KEYWORD = "not_analyzed"; // ES 5: change to true and rename to INDEX_SEARCHABLE - public static final String INDEX_NOT_SEARCHABLE = "no"; // ES 5: change to false - public static final String FIELD_TYPE_TEXT = "string"; // ES 5: change to text - public static final String FIELD_TYPE_KEYWORD = "string"; // ES 5: change to keyword + public static final String INDEX_SEARCHABLE = "true"; + public static final String INDEX_NOT_SEARCHABLE = "false"; + public static final String FIELD_TYPE_TEXT = "text"; + public static final String FIELD_TYPE_KEYWORD = "keyword"; public static final String STANDARD = "standard"; public static final String PATTERN = "pattern"; public static final String CUSTOM = "custom"; public static final String KEYWORD = "keyword"; public static final String CLASSIC = "classic"; - public static final boolean REFRESH_IMMEDIATE = true; // ES 5: replace with org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE - public static final boolean REFRESH_NONE = false; // ES 5: replace with org.elasticsearch.action.support.WriteRequest.RefreshPolicy.NONE + public static final RefreshPolicy REFRESH_IMMEDIATE = RefreshPolicy.IMMEDIATE; + public static final RefreshPolicy REFRESH_NONE = RefreshPolicy.NONE; public static final String TRUNCATE = "truncate"; diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/DefaultIndexSettingsElement.java b/server/sonar-server/src/main/java/org/sonar/server/es/DefaultIndexSettingsElement.java index f4de5e0e5e7b..e8e3610d0fe6 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/DefaultIndexSettingsElement.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/DefaultIndexSettingsElement.java @@ -33,7 +33,7 @@ import static org.sonar.server.es.DefaultIndexSettings.FIELD_TYPE_TEXT; import static org.sonar.server.es.DefaultIndexSettings.FILTER; import static org.sonar.server.es.DefaultIndexSettings.INDEX; -import static org.sonar.server.es.DefaultIndexSettings.INDEX_SEARCHABLE_FOR_TEXT; +import static org.sonar.server.es.DefaultIndexSettings.INDEX_SEARCHABLE; import static org.sonar.server.es.DefaultIndexSettings.KEYWORD; import static org.sonar.server.es.DefaultIndexSettings.LOWERCASE; import static org.sonar.server.es.DefaultIndexSettings.MAXIMUM_NGRAM_LENGTH; @@ -125,7 +125,7 @@ protected void setup() { public SortedMap fieldMapping() { return ImmutableSortedMap.of( TYPE, FIELD_TYPE_TEXT, - INDEX, INDEX_SEARCHABLE_FOR_TEXT, + INDEX, INDEX_SEARCHABLE, ANALYZER, getName()); } }, @@ -149,7 +149,7 @@ protected void setup() { public SortedMap fieldMapping() { return ImmutableSortedMap.of( TYPE, FIELD_TYPE_TEXT, - INDEX, INDEX_SEARCHABLE_FOR_TEXT, + INDEX, INDEX_SEARCHABLE, ANALYZER, INDEX_GRAMS_ANALYZER.getName(), SEARCH_ANALYZER, getName()); } @@ -174,7 +174,7 @@ protected void setup() { public SortedMap fieldMapping() { return ImmutableSortedMap.of( TYPE, FIELD_TYPE_TEXT, - INDEX, INDEX_SEARCHABLE_FOR_TEXT, + INDEX, INDEX_SEARCHABLE, ANALYZER, INDEX_PREFIX_ANALYZER.getName(), SEARCH_ANALYZER, getName()); } @@ -199,7 +199,7 @@ protected void setup() { public SortedMap fieldMapping() { return ImmutableSortedMap.of( TYPE, FIELD_TYPE_TEXT, - INDEX, INDEX_SEARCHABLE_FOR_TEXT, + INDEX, INDEX_SEARCHABLE, ANALYZER, INDEX_PREFIX_CASE_INSENSITIVE_ANALYZER.getName(), SEARCH_ANALYZER, getName()); } @@ -224,7 +224,7 @@ protected void setup() { public SortedMap fieldMapping() { return ImmutableSortedMap.of( TYPE, FIELD_TYPE_TEXT, - INDEX, INDEX_SEARCHABLE_FOR_TEXT, + INDEX, INDEX_SEARCHABLE, ANALYZER, USER_INDEX_GRAMS_ANALYZER.getName(), SEARCH_ANALYZER, getName()); } @@ -249,7 +249,7 @@ protected void setup() { public SortedMap fieldMapping() { return ImmutableSortedMap.of( TYPE, FIELD_TYPE_TEXT, - INDEX, INDEX_SEARCHABLE_FOR_TEXT, + INDEX, INDEX_SEARCHABLE, ANALYZER, INDEX_WORDS_ANALYZER.getName(), SEARCH_ANALYZER, getName()); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/EsClientProvider.java b/server/sonar-server/src/main/java/org/sonar/server/es/EsClientProvider.java index d4259c676598..8a2f6ed18c32 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/EsClientProvider.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/EsClientProvider.java @@ -23,6 +23,7 @@ import java.net.InetAddress; import java.net.UnknownHostException; import java.util.Arrays; +import java.util.Collections; import java.util.stream.Collectors; import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.common.transport.InetSocketTransportAddress; @@ -54,13 +55,13 @@ public EsClient provide(Configuration config) { boolean clusterEnabled = config.getBoolean(ProcessProperties.CLUSTER_ENABLED).orElse(false); if (clusterEnabled && config.getBoolean(ProcessProperties.CLUSTER_SEARCH_DISABLED).orElse(false)) { esSettings.put("client.transport.sniff", true); - nativeClient = TransportClient.builder().settings(esSettings).build(); + nativeClient = new TransportClient(esSettings.build(), Collections.emptyList()) {}; Arrays.stream(config.getStringArray(ProcessProperties.CLUSTER_SEARCH_HOSTS)) .map(HostAndPort::fromString) .forEach(h -> addHostToClient(h, nativeClient)); LOGGER.info("Connected to remote Elasticsearch: [{}]", displayedAddresses(nativeClient)); } else { - nativeClient = TransportClient.builder().settings(esSettings).build(); + nativeClient = new TransportClient(esSettings.build(), Collections.emptyList()) {}; HostAndPort host = HostAndPort.fromParts(config.get(ProcessProperties.SEARCH_HOST).get(), config.getInt(ProcessProperties.SEARCH_PORT).get()); addHostToClient(host, nativeClient); LOGGER.info("Connected to local Elasticsearch: [{}]", displayedAddresses(nativeClient)); diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java b/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java index 261251e1d136..a4fb706fd997 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java @@ -43,8 +43,7 @@ import static org.sonar.server.es.DefaultIndexSettings.FIELD_TYPE_TEXT; import static org.sonar.server.es.DefaultIndexSettings.INDEX; import static org.sonar.server.es.DefaultIndexSettings.INDEX_NOT_SEARCHABLE; -import static org.sonar.server.es.DefaultIndexSettings.INDEX_SEARCHABLE_FOR_KEYWORD; -import static org.sonar.server.es.DefaultIndexSettings.INDEX_SEARCHABLE_FOR_TEXT; +import static org.sonar.server.es.DefaultIndexSettings.INDEX_SEARCHABLE; import static org.sonar.server.es.DefaultIndexSettings.TYPE; import static org.sonar.server.es.DefaultIndexSettingsElement.UUID_MODULE_ANALYZER; @@ -180,7 +179,7 @@ public NewIndexType createShortField(String fieldName) { public NewIndexType createUuidPathField(String fieldName) { return setProperty(fieldName, ImmutableSortedMap.of( TYPE, FIELD_TYPE_TEXT, - INDEX, INDEX_SEARCHABLE_FOR_TEXT, + INDEX, DefaultIndexSettings.INDEX_SEARCHABLE, ANALYZER, UUID_MODULE_ANALYZER.getName())); } @@ -248,12 +247,6 @@ public KeywordFieldBuilder termVectorWithPositionOffsets() { } /** - * "index: no" -> Don’t index this field at all. This field will not be searchable. - * By default field is "not_analyzed": it is searchable, but index the value exactly - * as specified. - */ - // ES 5: update javadoc to: - /* * "index: false" -> Make this field not searchable. * By default field is "true": it is searchable, but index the value exactly * as specified. @@ -268,10 +261,8 @@ public NewIndexType build() { if (subFields.isEmpty()) { hash.putAll(ImmutableMap.of( "type", FIELD_TYPE_KEYWORD, - "index", disableSearch ? INDEX_NOT_SEARCHABLE : INDEX_SEARCHABLE_FOR_KEYWORD, - "norms", - ImmutableMap.of("enabled", String.valueOf(!disableNorms)) // ES 5: replace with String.valueOf(!disableNorms) - )); + "index", disableSearch ? INDEX_NOT_SEARCHABLE : INDEX_SEARCHABLE, + "norms", String.valueOf(!disableNorms))); } else { hash.put("type", "multi_field"); @@ -291,10 +282,10 @@ public NewIndexType build() { multiFields.put(fieldName, ImmutableMap.of( "type", FIELD_TYPE_KEYWORD, - "index", INDEX_SEARCHABLE_FOR_KEYWORD, + "index", INDEX_SEARCHABLE, "term_vector", termVectorWithPositionOffsets ? "with_positions_offsets" : "no", "norms", - ImmutableMap.of("enabled", "false") // ES 5: replace with "false" + "false" )); hash.put("fields", multiFields); @@ -329,7 +320,7 @@ private NestedFieldBuilder setProperty(String fieldName, Object value) { public NestedFieldBuilder addKeywordField(String fieldName) { return setProperty(fieldName, ImmutableMap.of( "type", FIELD_TYPE_KEYWORD, - "index", INDEX_SEARCHABLE_FOR_KEYWORD)); + "index", INDEX_SEARCHABLE)); } public NestedFieldBuilder addDoubleField(String fieldName) { diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/Sorting.java b/server/sonar-server/src/main/java/org/sonar/server/es/Sorting.java index 082664fec0f2..72501d7d4e57 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/Sorting.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/Sorting.java @@ -23,10 +23,10 @@ import com.google.common.collect.ListMultimap; import com.google.common.collect.Lists; import java.util.List; -import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortOrder; +import org.sonar.core.util.stream.MoreCollectors; import static org.sonar.server.ws.WsUtils.checkRequest; @@ -61,25 +61,25 @@ public List getFields(String name) { return fields.get(name); } - public void fill(SearchRequestBuilder request, String name, boolean asc) { + public List fill(String name, boolean asc) { List list = fields.get(name); checkRequest(!list.isEmpty(), "Bad sort field: %s", name); - doFill(request, list, asc); + return doFill(list, asc); } - public void fillDefault(SearchRequestBuilder request) { - doFill(request, defaultFields, true); + public List fillDefault() { + return doFill(defaultFields, true); } - private static void doFill(SearchRequestBuilder request, List fields, boolean asc) { - for (Field field : fields) { + private static List doFill(List fields, boolean asc) { + return fields.stream().map(field -> { FieldSortBuilder sortBuilder = SortBuilders.fieldSort(field.name); boolean effectiveAsc = asc != field.reverse; sortBuilder.order(effectiveAsc ? SortOrder.ASC : SortOrder.DESC); boolean effectiveMissingLast = asc == field.missingLast; sortBuilder.missing(effectiveMissingLast ? "_last" : "_first"); - request.addSort(sortBuilder); - } + return sortBuilder; + }).collect(MoreCollectors.toList(fields.size())); } public static class Field { diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/StickyFacetBuilder.java b/server/sonar-server/src/main/java/org/sonar/server/es/StickyFacetBuilder.java index e8e398929d69..b714d3bbfbc0 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/StickyFacetBuilder.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/StickyFacetBuilder.java @@ -35,7 +35,8 @@ import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.bucket.terms.Terms.Order; -import org.elasticsearch.search.aggregations.bucket.terms.TermsBuilder; +import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; @@ -80,7 +81,7 @@ public AggregationBuilder buildStickyFacet(String fieldName, String facetName, O * @param selected the terms, that the user already has selected * @return the (global) aggregation, that can be added on top level of the elasticsearch request */ - public AggregationBuilder buildStickyFacet(String fieldName, String facetName, Function> additionalAggregationFilter, Object... selected) { + public AggregationBuilder buildStickyFacet(String fieldName, String facetName, Function additionalAggregationFilter, Object... selected) { return buildStickyFacet(fieldName, facetName, FACET_DEFAULT_SIZE, additionalAggregationFilter, selected); } @@ -88,7 +89,7 @@ public AggregationBuilder buildStickyFacet(String fieldName, String facetName, i return buildStickyFacet(fieldName, facetName, size, t -> t, selected); } - private AggregationBuilder buildStickyFacet(String fieldName, String facetName, int size, Function> additionalAggregationFilter, + private AggregationBuilder buildStickyFacet(String fieldName, String facetName, int size, Function additionalAggregationFilter, Object... selected) { BoolQueryBuilder facetFilter = getStickyFacetFilter(fieldName); FilterAggregationBuilder facetTopAggregation = buildTopFacetAggregation(fieldName, facetName, facetFilter, size, additionalAggregationFilter); @@ -114,17 +115,16 @@ public FilterAggregationBuilder buildTopFacetAggregation(String fieldName, Strin } private FilterAggregationBuilder buildTopFacetAggregation(String fieldName, String facetName, BoolQueryBuilder facetFilter, int size, - Function> additionalAggregationFilter) { - TermsBuilder termsAggregation = buildTermsFacetAggregation(fieldName, facetName, size); - AggregationBuilder improvedAggregation = additionalAggregationFilter.apply(termsAggregation); + Function additionalAggregationFilter) { + TermsAggregationBuilder termsAggregation = buildTermsFacetAggregation(fieldName, facetName, size); + AggregationBuilder improvedAggregation = additionalAggregationFilter.apply(termsAggregation); return AggregationBuilders - .filter(facetName + "_filter") - .filter(facetFilter) + .filter(facetName + "_filter", facetFilter) .subAggregation(improvedAggregation); } - private TermsBuilder buildTermsFacetAggregation(String fieldName, String facetName, int size) { - TermsBuilder termsAggregation = AggregationBuilders.terms(facetName) + private TermsAggregationBuilder buildTermsFacetAggregation(String fieldName, String facetName, int size) { + TermsAggregationBuilder termsAggregation = AggregationBuilders.terms(facetName) .field(fieldName) .order(order) .size(size) @@ -136,7 +136,7 @@ private TermsBuilder buildTermsFacetAggregation(String fieldName, String facetNa } public FilterAggregationBuilder addSelectedItemsToFacet(String fieldName, String facetName, FilterAggregationBuilder facetTopAggregation, - Function> additionalAggregationFilter, Object... selected) { + Function additionalAggregationFilter, Object... selected) { if (selected.length <= 0) { return facetTopAggregation; } @@ -145,14 +145,14 @@ public FilterAggregationBuilder addSelectedItemsToFacet(String fieldName, String .map(s -> EsUtils.escapeSpecialRegexChars(s.toString())) .collect(PIPE_JOINER); - TermsBuilder selectedTerms = AggregationBuilders.terms(facetName + "_selected") + TermsAggregationBuilder selectedTerms = AggregationBuilders.terms(facetName + "_selected") .field(fieldName) - .include(includes); + .includeExclude(new IncludeExclude(includes, null)); if (subAggregation != null) { selectedTerms = selectedTerms.subAggregation(subAggregation); } - AggregationBuilder improvedAggregation = additionalAggregationFilter.apply(selectedTerms); + AggregationBuilder improvedAggregation = additionalAggregationFilter.apply(selectedTerms); facetTopAggregation.subAggregation(improvedAggregation); return facetTopAggregation; } diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/metadata/MetadataIndex.java b/server/sonar-server/src/main/java/org/sonar/server/es/metadata/MetadataIndex.java index f2ab6d4cacd1..4b0d70f68c56 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/metadata/MetadataIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/metadata/MetadataIndex.java @@ -61,7 +61,8 @@ private static String initializedId(IndexType indexType) { } private Optional getMetadata(String id) { - GetRequestBuilder request = esClient.prepareGet(MetadataIndexDefinition.INDEX_TYPE_METADATA, id).setFields(MetadataIndexDefinition.FIELD_VALUE); + GetRequestBuilder request = esClient.prepareGet(MetadataIndexDefinition.INDEX_TYPE_METADATA, id) + .setStoredFields(MetadataIndexDefinition.FIELD_VALUE); GetResponse response = request.get(); if (response.isExists()) { GetField field = response.getField(MetadataIndexDefinition.FIELD_VALUE); @@ -75,7 +76,7 @@ private void setMetadata(String hash, String id) { esClient.prepareIndex(MetadataIndexDefinition.INDEX_TYPE_METADATA) .setId(id) .setSource(MetadataIndexDefinition.FIELD_VALUE, hash) - .setRefresh(REFRESH_IMMEDIATE) // ES 5: change to setRefreshPolicy + .setRefreshPolicy(REFRESH_IMMEDIATE) .get(); } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxySearchRequestBuilder.java b/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxySearchRequestBuilder.java index 6c834d4a7109..0f3fdb066bbe 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxySearchRequestBuilder.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxySearchRequestBuilder.java @@ -19,7 +19,6 @@ */ package org.sonar.server.es.request; -import java.io.IOException; import java.util.Arrays; import org.elasticsearch.action.ListenableActionFuture; import org.elasticsearch.action.search.SearchAction; @@ -27,9 +26,6 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; import org.sonar.api.utils.log.Profiler; import org.sonar.server.es.EsClient; @@ -71,7 +67,7 @@ public ListenableActionFuture execute() { @Override public String toString() { StringBuilder message = new StringBuilder(); - message.append(String.format("ES search request '%s'", xContentToString(super.internalBuilder()))); + message.append(String.format("ES search request '%s'", super.request)); if (request.indices().length > 0) { message.append(String.format(" on indices '%s'", Arrays.toString(request.indices()))); } @@ -80,15 +76,4 @@ public String toString() { } return message.toString(); } - - private static String xContentToString(ToXContent toXContent) { - try { - XContentBuilder builder = XContentFactory.jsonBuilder(); - toXContent.toXContent(builder, ToXContent.EMPTY_PARAMS); - return builder.string(); - } catch (IOException e) { - throw new IllegalStateException("Fail to convert request to string", e); - } - } - } diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java index 3fb083766d93..59e54f1b91b1 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java @@ -43,16 +43,21 @@ import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.indices.TermsLookup; import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; +import org.elasticsearch.search.aggregations.bucket.histogram.ExtendedBounds; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.bucket.terms.Terms.Order; -import org.elasticsearch.search.aggregations.bucket.terms.TermsBuilder; +import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude; import org.elasticsearch.search.aggregations.metrics.max.InternalMax; import org.elasticsearch.search.aggregations.metrics.min.Min; -import org.elasticsearch.search.aggregations.metrics.sum.SumBuilder; +import org.elasticsearch.search.aggregations.metrics.sum.SumAggregationBuilder; +import org.elasticsearch.search.sort.FieldSortBuilder; +import org.joda.time.DateTimeZone; import org.elasticsearch.search.aggregations.metrics.valuecount.InternalValueCount; import org.joda.time.Duration; import org.sonar.api.utils.DateUtils; @@ -132,7 +137,7 @@ public class IssueIndex { // TODO move to Facets ? private static final String FACET_SUFFIX_MISSING = "_missing"; private static final String IS_ASSIGNED_FILTER = "__isAssigned"; - private static final SumBuilder EFFORT_AGGREGATION = AggregationBuilders.sum(FACET_MODE_EFFORT).field(IssueIndexDefinition.FIELD_ISSUE_EFFORT); + private static final SumAggregationBuilder EFFORT_AGGREGATION = AggregationBuilders.sum(FACET_MODE_EFFORT).field(IssueIndexDefinition.FIELD_ISSUE_EFFORT); private static final Order EFFORT_AGGREGATION_ORDER = Order.aggregation(FACET_MODE_EFFORT, false); private static final int DEFAULT_FACET_SIZE = 15; private static final Duration TWENTY_DAYS = Duration.standardDays(20L); @@ -222,11 +227,12 @@ private static QueryBuilder createViewFilter(Collection viewUuids) { BoolQueryBuilder viewsFilter = boolQuery(); for (String viewUuid : viewUuids) { - viewsFilter.should(QueryBuilders.termsLookupQuery(IssueIndexDefinition.FIELD_ISSUE_PROJECT_UUID) - .lookupIndex(ViewIndexDefinition.INDEX_TYPE_VIEW.getIndex()) - .lookupType(ViewIndexDefinition.INDEX_TYPE_VIEW.getType()) - .lookupId(viewUuid) - .lookupPath(ViewIndexDefinition.FIELD_PROJECTS)); + viewsFilter.should(QueryBuilders.termsLookupQuery(IssueIndexDefinition.FIELD_ISSUE_PROJECT_UUID, + new TermsLookup( + ViewIndexDefinition.INDEX_TYPE_VIEW.getIndex(), + ViewIndexDefinition.INDEX_TYPE_VIEW.getType(), + viewUuid, + ViewIndexDefinition.FIELD_PROJECTS))); } return viewsFilter; } @@ -352,13 +358,16 @@ public SearchResponse search(IssueQuery query, SearchOptions options) { } private void configureSorting(IssueQuery query, SearchRequestBuilder esRequest) { + createSortBuilders(query).forEach(esRequest::addSort); + } + + private List createSortBuilders(IssueQuery query) { String sortField = query.sort(); if (sortField != null) { boolean asc = BooleanUtils.isTrue(query.asc()); - sorting.fill(esRequest, sortField, asc); - } else { - sorting.fillDefault(esRequest); + return sorting.fill(sortField, asc); } + return sorting.fillDefault(); } private Map createFilters(IssueQuery query) { @@ -523,13 +532,13 @@ private Optional getCreatedAtFacet(IssueQuery query, Map listTags(OrganizationDto organization, @Nullable String text .filter(termQuery(FIELD_ISSUE_ORGANIZATION_UUID, organization.getUuid()))) .setSize(0); - TermsBuilder termsAggregation = AggregationBuilders.terms(AGGREGATION_NAME_FOR_TAGS) + TermsAggregationBuilder termsAggregation = AggregationBuilders.terms(AGGREGATION_NAME_FOR_TAGS) .field(IssueIndexDefinition.FIELD_ISSUE_TAGS) .size(size) .order(Terms.Order.term(true)) .minDocCount(1L); if (textQuery != null) { String escapedTextQuery = escapeSpecialRegexChars(textQuery); - termsAggregation.include(format(SUBSTRING_MATCH_REGEXP, escapedTextQuery)); + termsAggregation.includeExclude(new IncludeExclude(format(SUBSTRING_MATCH_REGEXP, escapedTextQuery), null)); } requestBuilder.addAggregation(termsAggregation); @@ -630,13 +638,13 @@ private Terms listTermsMatching(String fieldName, IssueQuery query, @Nullable St requestBuilder.setQuery(boolQuery().must(QueryBuilders.matchAllQuery()).filter(createBoolFilter(query))); - TermsBuilder aggreg = AggregationBuilders.terms("_ref") + TermsAggregationBuilder aggreg = AggregationBuilders.terms("_ref") .field(fieldName) .size(maxNumberOfTags) .order(termsOrder) .minDocCount(1L); if (textQuery != null) { - aggreg.include(format(SUBSTRING_MATCH_REGEXP, escapeSpecialRegexChars(textQuery))); + aggreg.includeExclude(new IncludeExclude(format(SUBSTRING_MATCH_REGEXP, escapeSpecialRegexChars(textQuery)), null)); } SearchResponse searchResponse = requestBuilder.addAggregation(aggreg).get(); @@ -671,8 +679,7 @@ public List searchProjectStatistics(List projectUuids long from = froms.get(i); request .addAggregation(AggregationBuilders - .filter(projectUuid) - .filter(boolQuery() + .filter(projectUuid, boolQuery() .filter(termQuery(IssueIndexDefinition.FIELD_ISSUE_PROJECT_UUID, projectUuid)) .filter(rangeQuery(IssueIndexDefinition.FIELD_ISSUE_FUNC_CREATED_AT).gte(new Date(from))) ) diff --git a/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndex.java b/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndex.java index ecc3da783778..047c8daea557 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndex.java @@ -30,16 +30,17 @@ import java.util.Set; import java.util.stream.IntStream; import javax.annotation.Nullable; +import org.apache.lucene.search.join.ScoreMode; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.search.aggregations.AbstractAggregationBuilder; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation.Bucket; -import org.elasticsearch.search.aggregations.bucket.filters.FiltersAggregationBuilder; -import org.elasticsearch.search.aggregations.bucket.range.RangeBuilder; +import org.elasticsearch.search.aggregations.bucket.range.RangeAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.terms.Terms; -import org.elasticsearch.search.aggregations.bucket.terms.TermsBuilder; +import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude; import org.elasticsearch.search.sort.FieldSortBuilder; import org.sonar.core.util.stream.MoreCollectors; import org.sonar.server.es.DefaultIndexSettingsElement; @@ -192,8 +193,7 @@ private static void addRangeFacet(SearchRequestBuilder esSearch, String metricKe private static void addRangeFacetIncludingNoData(SearchRequestBuilder esSearch, String metricKey, StickyFacetBuilder facetBuilder, Double... thresholds) { esSearch.addAggregation(createStickyFacet(metricKey, facetBuilder, - AggregationBuilders.filter("combined_" + metricKey) - .filter(matchAllQuery()) + AggregationBuilders.filter("combined_" + metricKey, matchAllQuery()) .subAggregation(createRangeFacet(metricKey, thresholds)) .subAggregation(createNoDataFacet(metricKey)))); } @@ -224,13 +224,14 @@ private static AbstractAggregationBuilder createStickyFacet(String facetKey, Sti BoolQueryBuilder facetFilter = facetBuilder.getStickyFacetFilter(facetKey); return AggregationBuilders .global(facetKey) - .subAggregation(AggregationBuilders.filter("facet_filter_" + facetKey) - .filter(facetFilter) - .subAggregation(aggregationBuilder)); + .subAggregation( + AggregationBuilders + .filter("facet_filter_" + facetKey, facetFilter) + .subAggregation(aggregationBuilder)); } private static AbstractAggregationBuilder createRangeFacet(String metricKey, Double... thresholds) { - RangeBuilder rangeAgg = AggregationBuilders.range(metricKey) + RangeAggregationBuilder rangeAgg = AggregationBuilders.range(metricKey) .field(FIELD_MEASURES_VALUE); final int lastIndex = thresholds.length - 1; IntStream.range(0, thresholds.length) @@ -245,38 +246,36 @@ private static AbstractAggregationBuilder createRangeFacet(String metricKey, Dou } }); - return AggregationBuilders.nested("nested_" + metricKey) - .path(FIELD_MEASURES) + return AggregationBuilders.nested("nested_" + metricKey, FIELD_MEASURES) .subAggregation( - AggregationBuilders.filter("filter_" + metricKey) - .filter(termsQuery(FIELD_MEASURES_KEY, metricKey)) + AggregationBuilders.filter("filter_" + metricKey, termsQuery(FIELD_MEASURES_KEY, metricKey)) .subAggregation(rangeAgg)); } private static AbstractAggregationBuilder createNoDataFacet(String metricKey) { - return AggregationBuilders.filter("no_data_" + metricKey) - .filter(boolQuery() - .mustNot(nestedQuery(FIELD_MEASURES, termQuery(FIELD_MEASURES_KEY, metricKey)))); + return AggregationBuilders.filter( + "no_data_" + metricKey, + boolQuery().mustNot(nestedQuery(FIELD_MEASURES, termQuery(FIELD_MEASURES_KEY, metricKey), ScoreMode.Avg))); } private static AbstractAggregationBuilder createRatingFacet(String metricKey) { - return AggregationBuilders.nested("nested_" + metricKey) - .path(FIELD_MEASURES) + return AggregationBuilders.nested("nested_" + metricKey, FIELD_MEASURES) .subAggregation( - AggregationBuilders.filter("filter_" + metricKey) - .filter(termsQuery(FIELD_MEASURES_KEY, metricKey)) - .subAggregation(filters(metricKey) - .filter("1", termQuery(FIELD_MEASURES_VALUE, 1d)) - .filter("2", termQuery(FIELD_MEASURES_VALUE, 2d)) - .filter("3", termQuery(FIELD_MEASURES_VALUE, 3d)) - .filter("4", termQuery(FIELD_MEASURES_VALUE, 4d)) - .filter("5", termQuery(FIELD_MEASURES_VALUE, 5d)))); + AggregationBuilders.filter("filter_" + metricKey, termsQuery(FIELD_MEASURES_KEY, metricKey)) + .subAggregation(filters(metricKey, + termQuery(FIELD_MEASURES_VALUE, 1d), + termQuery(FIELD_MEASURES_VALUE, 2d), + termQuery(FIELD_MEASURES_VALUE, 3d), + termQuery(FIELD_MEASURES_VALUE, 4d), + termQuery(FIELD_MEASURES_VALUE, 5d)))); } private static AbstractAggregationBuilder createQualityGateFacet() { - FiltersAggregationBuilder qualityGateStatusFilter = AggregationBuilders.filters(ALERT_STATUS_KEY); - QUALITY_GATE_STATUS.entrySet().forEach(entry -> qualityGateStatusFilter.filter(entry.getKey(), termQuery(FIELD_QUALITY_GATE_STATUS, entry.getValue()))); - return qualityGateStatusFilter; + return AggregationBuilders.filters( + ALERT_STATUS_KEY, + QUALITY_GATE_STATUS.entrySet().stream() + .map(entry -> termQuery(FIELD_QUALITY_GATE_STATUS, entry.getValue())) + .toArray(QueryBuilder[]::new)); } private Map createFilters(ProjectMeasuresQuery query) { @@ -322,11 +321,18 @@ private static Optional createTextQueryFilter(ProjectMeasuresQuery private static QueryBuilder toQuery(MetricCriterion criterion) { if (criterion.isNoData()) { - return boolQuery().mustNot(nestedQuery(FIELD_MEASURES, termQuery(FIELD_MEASURES_KEY, criterion.getMetricKey()))); + return boolQuery().mustNot( + nestedQuery( + FIELD_MEASURES, + termQuery(FIELD_MEASURES_KEY, criterion.getMetricKey()), + ScoreMode.Avg)); } - return nestedQuery(FIELD_MEASURES, boolQuery() - .filter(termQuery(FIELD_MEASURES_KEY, criterion.getMetricKey())) - .filter(toValueQuery(criterion))); + return nestedQuery( + FIELD_MEASURES, + boolQuery() + .filter(termQuery(FIELD_MEASURES_KEY, criterion.getMetricKey())) + .filter(toValueQuery(criterion)), + ScoreMode.Avg); } private static QueryBuilder toValueQuery(MetricCriterion criterion) { @@ -355,13 +361,13 @@ public List searchTags(@Nullable String textQuery, int size) { return emptyList(); } - TermsBuilder tagFacet = AggregationBuilders.terms(FIELD_TAGS) + TermsAggregationBuilder tagFacet = AggregationBuilders.terms(FIELD_TAGS) .field(FIELD_TAGS) .size(size) .minDocCount(1) .order(Terms.Order.term(true)); if (textQuery != null) { - tagFacet.include(".*" + escapeSpecialRegexChars(textQuery) + ".*"); + tagFacet.includeExclude(new IncludeExclude(".*" + escapeSpecialRegexChars(textQuery) + ".*", null)); } SearchRequestBuilder searchQuery = client diff --git a/server/sonar-server/src/main/java/org/sonar/server/permission/index/AuthorizationTypeSupport.java b/server/sonar-server/src/main/java/org/sonar/server/permission/index/AuthorizationTypeSupport.java index f1ba4e881dd0..991e6a4233de 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/permission/index/AuthorizationTypeSupport.java +++ b/server/sonar-server/src/main/java/org/sonar/server/permission/index/AuthorizationTypeSupport.java @@ -117,7 +117,9 @@ public QueryBuilder createQueryFilter() { .map(GroupDto::getId) .forEach(groupId -> filter.should(termQuery(FIELD_GROUP_IDS, groupId))); - return QueryBuilders.hasParentQuery(TYPE_AUTHORIZATION, - QueryBuilders.boolQuery().filter(filter)); + return QueryBuilders.hasParentQuery( + TYPE_AUTHORIZATION, + QueryBuilders.boolQuery().filter(filter), + false); } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/platform/BackendCleanup.java b/server/sonar-server/src/main/java/org/sonar/server/platform/BackendCleanup.java index a508d4de517d..9274f1e53bca 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/platform/BackendCleanup.java +++ b/server/sonar-server/src/main/java/org/sonar/server/platform/BackendCleanup.java @@ -95,7 +95,7 @@ public void clearIndexes() { try { esClient.prepareClearCache().get(); - for (String index : esClient.prepareState().get().getState().getMetaData().concreteAllIndices()) { + for (String index : esClient.prepareState().get().getState().getMetaData().getConcreteAllIndices()) { clearIndex(new IndexType(index, index)); } } catch (Exception e) { diff --git a/server/sonar-server/src/main/java/org/sonar/server/platform/monitoring/EsMonitor.java b/server/sonar-server/src/main/java/org/sonar/server/platform/monitoring/EsMonitor.java index 204a59bd27fd..1c64ade4c1bb 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/platform/monitoring/EsMonitor.java +++ b/server/sonar-server/src/main/java/org/sonar/server/platform/monitoring/EsMonitor.java @@ -111,13 +111,13 @@ private LinkedHashMap> nodeAttributes() { nodes.put(node.getName(), nodeAttributes); nodeAttributes.put("Address", node.getAddress().toString()); nodeAttributes.put("Type", node.isMasterNode() ? "Master" : "Slave"); - nodeAttributes.put("Disk Available", byteCountToDisplaySize(stats.getFs().getTotal().getAvailable().bytes())); + nodeAttributes.put("Disk Available", byteCountToDisplaySize(stats.getFs().getTotal().getAvailable().getBytes())); nodeAttributes.put("Store Size", byteCountToDisplaySize(stats.getIndices().getStore().getSizeInBytes())); nodeAttributes.put("Open Files", stats.getProcess().getOpenFileDescriptors()); nodeAttributes.put("JVM Heap Usage", formatPercent(stats.getJvm().getMem().getHeapUsedPercent())); - nodeAttributes.put("JVM Heap Used", byteCountToDisplaySize(stats.getJvm().getMem().getHeapUsed().bytes())); - nodeAttributes.put("JVM Heap Max", byteCountToDisplaySize(stats.getJvm().getMem().getHeapMax().bytes())); - nodeAttributes.put("JVM Non Heap Used", byteCountToDisplaySize(stats.getJvm().getMem().getNonHeapUsed().bytes())); + nodeAttributes.put("JVM Heap Used", byteCountToDisplaySize(stats.getJvm().getMem().getHeapUsed().getBytes())); + nodeAttributes.put("JVM Heap Max", byteCountToDisplaySize(stats.getJvm().getMem().getHeapMax().getBytes())); + nodeAttributes.put("JVM Non Heap Used", byteCountToDisplaySize(stats.getJvm().getMem().getNonHeapUsed().getBytes())); nodeAttributes.put("JVM Threads", stats.getJvm().getThreads().getCount()); nodeAttributes.put("Field Data Memory", byteCountToDisplaySize(stats.getIndices().getFieldData().getMemorySizeInBytes())); nodeAttributes.put("Field Data Circuit Breaker Limit", byteCountToDisplaySize(stats.getBreaker().getStats(CircuitBreaker.FIELDDATA).getLimit())); diff --git a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndex.java b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndex.java index e1774ec72e1d..c49f11dbe5fe 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndex.java @@ -30,22 +30,22 @@ import java.util.function.Function; import javax.annotation.Nullable; import org.apache.commons.lang.StringUtils; +import org.apache.lucene.search.join.ScoreMode; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.HasParentQueryBuilder; -import org.elasticsearch.index.query.MatchQueryBuilder; +import org.elasticsearch.index.query.Operator; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.index.query.SimpleQueryStringBuilder; import org.elasticsearch.index.query.TermsQueryBuilder; -import org.elasticsearch.search.aggregations.AbstractAggregationBuilder; import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.terms.Terms; -import org.elasticsearch.search.aggregations.bucket.terms.TermsBuilder; +import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude; import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortOrder; @@ -138,7 +138,7 @@ public SearchIdResult search(RuleQuery query, SearchOptions options) { Map filters = buildFilters(query); if (!options.getFacets().isEmpty()) { - for (AbstractAggregationBuilder aggregation : getFacets(query, options, qb, filters).values()) { + for (AggregationBuilder aggregation : getFacets(query, options, qb, filters).values()) { esSearch.addAggregation(aggregation); } } @@ -194,12 +194,12 @@ private static QueryBuilder buildQuery(RuleQuery query) { qb.should(simpleQueryStringQuery(query.getQueryText()) .field(SEARCH_WORDS_ANALYZER.subField(FIELD_RULE_NAME), 20f) .field(FIELD_RULE_HTML_DESCRIPTION, 3f) - .defaultOperator(SimpleQueryStringBuilder.Operator.AND)).boost(20f); + .defaultOperator(Operator.AND)).boost(20f); // Match and partial Match queries // Search by key uses the "sortable" sub-field as it requires to be case-insensitive (lower-case filtering) - qb.should(matchQuery(SORTABLE_ANALYZER.subField(FIELD_RULE_KEY), queryString).operator(MatchQueryBuilder.Operator.AND).boost(30f)); - qb.should(matchQuery(SORTABLE_ANALYZER.subField(FIELD_RULE_RULE_KEY), queryString).operator(MatchQueryBuilder.Operator.AND).boost(15f)); + qb.should(matchQuery(SORTABLE_ANALYZER.subField(FIELD_RULE_KEY), queryString).operator(Operator.AND).boost(30f)); + qb.should(matchQuery(SORTABLE_ANALYZER.subField(FIELD_RULE_RULE_KEY), queryString).operator(Operator.AND).boost(15f)); qb.should(termQuery(FIELD_RULE_LANGUAGE, queryString, 3f)); return qb; } @@ -207,7 +207,7 @@ private static QueryBuilder buildQuery(RuleQuery query) { private static QueryBuilder termQuery(String field, String query, float boost) { return QueryBuilders.multiMatchQuery(query, field, SEARCH_WORDS_ANALYZER.subField(field)) - .operator(MatchQueryBuilder.Operator.AND) + .operator(Operator.AND) .boost(boost); } @@ -258,7 +258,7 @@ private static Map buildFilters(RuleQuery query) { .map(tag -> boolQuery() .filter(QueryBuilders.termQuery(FIELD_RULE_EXTENSION_TAGS, tag)) .filter(termsQuery(FIELD_RULE_EXTENSION_SCOPE, RuleExtensionScope.system().getScope(), RuleExtensionScope.organization(query.getOrganization()).getScope()))) - .map(childQuery -> QueryBuilders.hasChildQuery(INDEX_TYPE_RULE_EXTENSION.getType(), childQuery)) + .map(childQuery -> QueryBuilders.hasChildQuery(INDEX_TYPE_RULE_EXTENSION.getType(), childQuery, ScoreMode.None)) .forEach(q::should); filters.put(FIELD_RULE_EXTENSION_TAGS, q); } @@ -315,18 +315,19 @@ private static Map buildFilters(RuleQuery query) { if (TRUE.equals(query.getActivation())) { filters.put("activation", QueryBuilders.hasChildQuery(INDEX_TYPE_ACTIVE_RULE.getType(), - childQuery)); + childQuery, ScoreMode.None)); } else if (FALSE.equals(query.getActivation())) { filters.put("activation", boolQuery().mustNot( QueryBuilders.hasChildQuery(INDEX_TYPE_ACTIVE_RULE.getType(), - childQuery))); + childQuery, ScoreMode.None))); } QProfileDto compareToQProfile = query.getCompareToQProfile(); if (compareToQProfile != null) { filters.put("comparison", QueryBuilders.hasChildQuery(INDEX_TYPE_ACTIVE_RULE.getType(), - boolQuery().must(QueryBuilders.termQuery(FIELD_ACTIVE_RULE_PROFILE_UUID, compareToQProfile.getRulesProfileUuid())))); + boolQuery().must(QueryBuilders.termQuery(FIELD_ACTIVE_RULE_PROFILE_UUID, compareToQProfile.getRulesProfileUuid())), + ScoreMode.None)); } } @@ -352,8 +353,8 @@ private static BoolQueryBuilder addTermFilter(BoolQueryBuilder filter, String fi return filter; } - private static Map getFacets(RuleQuery query, SearchOptions options, QueryBuilder queryBuilder, Map filters) { - Map aggregations = new HashMap<>(); + private static Map getFacets(RuleQuery query, SearchOptions options, QueryBuilder queryBuilder, Map filters) { + Map aggregations = new HashMap<>(); StickyFacetBuilder stickyFacetBuilder = stickyFacetBuilder(queryBuilder, filters); addDefaultFacets(query, options, aggregations, stickyFacetBuilder); @@ -369,7 +370,7 @@ private static Map getFacets(RuleQuery query return aggregations; } - private static void addDefaultFacets(RuleQuery query, SearchOptions options, Map aggregations, StickyFacetBuilder stickyFacetBuilder) { + private static void addDefaultFacets(RuleQuery query, SearchOptions options, Map aggregations, StickyFacetBuilder stickyFacetBuilder) { if (options.getFacets().contains(FACET_LANGUAGES) || options.getFacets().contains(FACET_OLD_DEFAULT)) { Collection languages = query.getLanguages(); aggregations.put(FACET_LANGUAGES, @@ -380,16 +381,16 @@ private static void addDefaultFacets(RuleQuery query, SearchOptions options, Map Collection tags = query.getTags(); checkArgument(query.getOrganization() != null, "Cannot use tags facet, if no organization is specified.", query.getTags()); - Function> childFeature = termsAggregation -> { + Function childFeature = termsAggregation -> { - FilterAggregationBuilder scopeAggregation = AggregationBuilders.filter("scope_filter_for_" + FACET_TAGS).filter( + FilterAggregationBuilder scopeAggregation = AggregationBuilders.filter( + "scope_filter_for_" + FACET_TAGS, termsQuery(FIELD_RULE_EXTENSION_SCOPE, RuleExtensionScope.system().getScope(), RuleExtensionScope.organization(query.getOrganization()).getScope())) .subAggregation(termsAggregation); - return AggregationBuilders.children("children_for_" + termsAggregation.getName()) - .childType(INDEX_TYPE_RULE_EXTENSION.getType()) + return AggregationBuilders.children("children_for_" + termsAggregation.getName(), INDEX_TYPE_RULE_EXTENSION.getType()) .subAggregation(scopeAggregation); }; @@ -411,24 +412,22 @@ private static void addDefaultFacets(RuleQuery query, SearchOptions options, Map } } - private static void addStatusFacetIfNeeded(SearchOptions options, Map aggregations, StickyFacetBuilder stickyFacetBuilder) { + private static void addStatusFacetIfNeeded(SearchOptions options, Map aggregations, StickyFacetBuilder stickyFacetBuilder) { if (options.getFacets().contains(FACET_STATUSES)) { BoolQueryBuilder facetFilter = stickyFacetBuilder.getStickyFacetFilter(FIELD_RULE_STATUS); - AbstractAggregationBuilder statuses = AggregationBuilders.filter(FACET_STATUSES + "_filter") - .filter(facetFilter) + AggregationBuilder statuses = AggregationBuilders.filter(FACET_STATUSES + "_filter", facetFilter) .subAggregation( AggregationBuilders .terms(FACET_STATUSES) .field(FIELD_RULE_STATUS) - .include(Joiner.on('|').join(ALL_STATUSES_EXCEPT_REMOVED)) - .exclude(RuleStatus.REMOVED.toString()) + .includeExclude(new IncludeExclude(Joiner.on('|').join(ALL_STATUSES_EXCEPT_REMOVED), RuleStatus.REMOVED.toString())) .size(ALL_STATUSES_EXCEPT_REMOVED.size())); aggregations.put(FACET_STATUSES, AggregationBuilders.global(FACET_STATUSES).subAggregation(statuses)); } } - private static void addActiveSeverityFacetIfNeeded(RuleQuery query, SearchOptions options, Map aggregations, + private static void addActiveSeverityFacetIfNeeded(RuleQuery query, SearchOptions options, Map aggregations, StickyFacetBuilder stickyFacetBuilder) { QProfileDto profile = query.getQProfile(); if (options.getFacets().contains(FACET_ACTIVE_SEVERITIES) && profile != null) { @@ -437,7 +436,8 @@ private static void addActiveSeverityFacetIfNeeded(RuleQuery query, SearchOption // from which we remove filters that concern active rules ("activation") HasParentQueryBuilder ruleFilter = QueryBuilders.hasParentQuery( INDEX_TYPE_RULE.getType(), - stickyFacetBuilder.getStickyFacetFilter("activation")); + stickyFacetBuilder.getStickyFacetFilter("activation"), + false); // Rebuilding the active rule filter without severities BoolQueryBuilder childrenFilter = boolQuery(); @@ -445,16 +445,15 @@ private static void addActiveSeverityFacetIfNeeded(RuleQuery query, SearchOption RuleIndex.addTermFilter(childrenFilter, FIELD_ACTIVE_RULE_INHERITANCE, query.getInheritance()); QueryBuilder activeRuleFilter = childrenFilter.must(ruleFilter); - AbstractAggregationBuilder activeSeverities = AggregationBuilders.children(FACET_ACTIVE_SEVERITIES + "_children") - .childType(INDEX_TYPE_ACTIVE_RULE.getType()) - .subAggregation(AggregationBuilders.filter(FACET_ACTIVE_SEVERITIES + "_filter") - .filter(activeRuleFilter) - .subAggregation( - AggregationBuilders - .terms(FACET_ACTIVE_SEVERITIES) - .field(FIELD_ACTIVE_RULE_SEVERITY) - .include(Joiner.on('|').join(Severity.ALL)) - .size(Severity.ALL.size()))); + AggregationBuilder activeSeverities = AggregationBuilders.children(FACET_ACTIVE_SEVERITIES + "_children", INDEX_TYPE_ACTIVE_RULE.getType()) + .subAggregation( + AggregationBuilders.filter(FACET_ACTIVE_SEVERITIES + "_filter", activeRuleFilter) + .subAggregation( + AggregationBuilders + .terms(FACET_ACTIVE_SEVERITIES) + .field(FIELD_ACTIVE_RULE_SEVERITY) + .includeExclude(new IncludeExclude(Joiner.on('|').join(Severity.ALL), null)) + .size(Severity.ALL.size()))); aggregations.put(FACET_ACTIVE_SEVERITIES, AggregationBuilders.global(FACET_ACTIVE_SEVERITIES).subAggregation(activeSeverities)); } @@ -501,12 +500,12 @@ public List terms(String fields) { } public List terms(String fields, @Nullable String query, int size) { - TermsBuilder termsAggregation = AggregationBuilders.terms(AGGREGATION_NAME) + TermsAggregationBuilder termsAggregation = AggregationBuilders.terms(AGGREGATION_NAME) .field(fields) .size(size) .minDocCount(1); if (query != null) { - termsAggregation.include(".*" + escapeSpecialRegexChars(query) + ".*"); + termsAggregation.includeExclude(new IncludeExclude(".*" + escapeSpecialRegexChars(query) + ".*", null)); } SearchRequestBuilder request = client .prepareSearch(INDEX_TYPE_RULE, INDEX_TYPE_ACTIVE_RULE) @@ -530,7 +529,7 @@ public List listTags(OrganizationDto organization, @Nullable String quer RuleExtensionScope.system().getScope(), RuleExtensionScope.organization(organization).getScope()); - TermsBuilder termsAggregation = AggregationBuilders.terms(AGGREGATION_NAME_FOR_TAGS) + TermsAggregationBuilder termsAggregation = AggregationBuilders.terms(AGGREGATION_NAME_FOR_TAGS) .field(FIELD_RULE_EXTENSION_TAGS) .size(size) .order(Terms.Order.term(true)) @@ -538,7 +537,8 @@ public List listTags(OrganizationDto organization, @Nullable String quer ofNullable(query) .map(EsUtils::escapeSpecialRegexChars) .map(queryString -> ".*" + queryString + ".*") - .ifPresent(termsAggregation::include); + .map(s -> new IncludeExclude(s, null)) + .ifPresent(termsAggregation::includeExclude); SearchRequestBuilder request = client .prepareSearch(INDEX_TYPE_RULE_EXTENSION) diff --git a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexDefinition.java b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexDefinition.java index ee85ac972ce7..bf563459590e 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexDefinition.java +++ b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexDefinition.java @@ -143,7 +143,7 @@ public void define(IndexDefinitionContext context) { ruleMapping.keywordFieldBuilder(FIELD_RULE_NAME).addSubFields(SORTABLE_ANALYZER, SEARCH_WORDS_ANALYZER).build(); ruleMapping.setProperty(FIELD_RULE_HTML_DESCRIPTION, ImmutableSortedMap.of( DefaultIndexSettings.TYPE, DefaultIndexSettings.FIELD_TYPE_TEXT, - DefaultIndexSettings.INDEX, DefaultIndexSettings.INDEX_SEARCHABLE_FOR_TEXT, + DefaultIndexSettings.INDEX, DefaultIndexSettings.INDEX_SEARCHABLE, DefaultIndexSettings.ANALYZER, ENGLISH_HTML_ANALYZER.getName(), DefaultIndexSettings.SEARCH_ANALYZER, ENGLISH_HTML_ANALYZER.getName())); ruleMapping.keywordFieldBuilder(FIELD_RULE_SEVERITY).disableNorms().build(); diff --git a/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndex.java b/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndex.java index c2fea95ae1e9..6e57de3be207 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndex.java @@ -22,6 +22,7 @@ import com.google.common.base.Optional; import java.util.ArrayList; import java.util.List; +import org.apache.lucene.search.join.ScoreMode; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.search.SearchHit; import org.sonar.server.es.EsClient; @@ -71,9 +72,12 @@ public SearchResult searchBySourceFileUuidAndLineNumber(String sourceFi SearchRequestBuilder searchRequest = client.prepareSearch(TestIndexDefinition.INDEX_TYPE_TEST) .setSize(searchOptions.getLimit()) .setFrom(searchOptions.getOffset()) - .setQuery(nestedQuery(FIELD_COVERED_FILES, boolQuery() - .must(termQuery(FIELD_COVERED_FILES + "." + FIELD_COVERED_FILE_UUID, sourceFileUuid)) - .must(termQuery(FIELD_COVERED_FILES + "." + FIELD_COVERED_FILE_LINES, lineNumber)))); + .setQuery(nestedQuery( + FIELD_COVERED_FILES, + boolQuery() + .must(termQuery(FIELD_COVERED_FILES + "." + FIELD_COVERED_FILE_UUID, sourceFileUuid)) + .must(termQuery(FIELD_COVERED_FILES + "." + FIELD_COVERED_FILE_LINES, lineNumber)), + ScoreMode.Avg)); return new SearchResult<>(searchRequest.get(), TestDoc::new); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndexDefinition.java b/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndexDefinition.java index 65cc11d58038..478b2702d4ac 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndexDefinition.java +++ b/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndexDefinition.java @@ -26,7 +26,7 @@ import org.sonar.server.es.NewIndex; import static org.sonar.server.es.DefaultIndexSettings.FIELD_TYPE_KEYWORD; -import static org.sonar.server.es.DefaultIndexSettings.INDEX_SEARCHABLE_FOR_KEYWORD; +import static org.sonar.server.es.DefaultIndexSettings.INDEX_SEARCHABLE; public class TestIndexDefinition implements IndexDefinition { @@ -68,7 +68,7 @@ public void define(IndexDefinitionContext context) { mapping.keywordFieldBuilder(FIELD_MESSAGE).disableNorms().disableSearch().build(); mapping.keywordFieldBuilder(FIELD_STACKTRACE).disableNorms().disableSearch().build(); mapping.setProperty(FIELD_COVERED_FILES, ImmutableMap.of("type", "nested", "properties", ImmutableMap.of( - FIELD_COVERED_FILE_UUID, ImmutableMap.of("type", FIELD_TYPE_KEYWORD, "index", INDEX_SEARCHABLE_FOR_KEYWORD), + FIELD_COVERED_FILE_UUID, ImmutableMap.of("type", FIELD_TYPE_KEYWORD, "index", INDEX_SEARCHABLE), FIELD_COVERED_FILE_LINES, ImmutableMap.of("type", "integer")))); mapping.createDateTimeField(FIELD_UPDATED_AT); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndex.java b/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndex.java index 8a848c79baf3..8eb0f0213e8f 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndex.java @@ -28,7 +28,7 @@ import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.MatchQueryBuilder; +import org.elasticsearch.index.query.Operator; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchHit; @@ -118,7 +118,7 @@ public SearchResult search(UserQuery userQuery, SearchOptions options) USER_SEARCH_GRAMS_ANALYZER.subField(FIELD_NAME), FIELD_EMAIL, USER_SEARCH_GRAMS_ANALYZER.subField(FIELD_EMAIL)) - .operator(MatchQueryBuilder.Operator.AND); + .operator(Operator.AND); } request.setQuery(boolQuery().must(esQuery).filter(filter)); diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/EsServerHolder.java b/server/sonar-server/src/test/java/org/sonar/server/es/EsServerHolder.java index 19947db83cbf..f0efb2a0e967 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/EsServerHolder.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/EsServerHolder.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.net.InetAddress; import java.nio.file.Files; +import java.util.Collections; import java.util.Properties; import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse; import org.elasticsearch.client.transport.TransportClient; @@ -72,10 +73,10 @@ public File getHomeDir() { } private void reset() { - TransportClient client = TransportClient.builder().settings(Settings.builder() + TransportClient client = new TransportClient(Settings.builder() .put("network.bind_host", "localhost") .put("cluster.name", clusterName) - .build()).build(); + .build(), Collections.emptyList()) {}; client.addTransportAddress(new InetSocketTransportAddress(InetAddress.getLoopbackAddress(), port)); // wait for node to be ready diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/EsTester.java b/server/sonar-server/src/test/java/org/sonar/server/es/EsTester.java index 567902214e22..6dccbe47bd44 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/EsTester.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/EsTester.java @@ -41,11 +41,11 @@ import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.node.Node; -import org.elasticsearch.node.NodeBuilder; +import org.elasticsearch.node.NodeValidationException; import org.elasticsearch.search.SearchHit; import org.junit.rules.ExternalResource; import org.sonar.api.config.internal.MapSettings; @@ -112,7 +112,7 @@ public void putDocuments(String index, String type, BaseDoc... docs) { public void putDocuments(IndexType indexType, BaseDoc... docs) { try { BulkRequestBuilder bulk = client.prepareBulk() - .setRefresh(REFRESH_IMMEDIATE); // ES 5: change to setRefreshPolicy + .setRefreshPolicy(REFRESH_IMMEDIATE); for (BaseDoc doc : docs) { bulk.add(new IndexRequest(indexType.getIndex(), indexType.getType(), doc.getId()) .parent(doc.getParent()) @@ -208,38 +208,45 @@ private static class NodeHolder { private final Node node; private NodeHolder() { + String nodeName = "tmp-es-" + RandomUtils.nextInt(); + Path tmpDir; try { - String nodeName = "tmp-es-" + RandomUtils.nextInt(); - Path tmpDir = Files.createTempDirectory("tmp-es"); - tmpDir.toFile().deleteOnExit(); - - node = NodeBuilder.nodeBuilder().local(true).data(true).settings(org.elasticsearch.common.settings.Settings.builder() - .put("cluster.name", nodeName) - .put("node.name", nodeName) - // the two following properties are probably not used because they are - // declared on indices too - .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) - // limit the number of threads created (see org.elasticsearch.common.util.concurrent.EsExecutors) - .put("processors", 1) - .put("http.enabled", false) - .put("config.ignore_system_properties", true) - .put("action.auto_create_index", false) - .put("path.home", tmpDir)) - .build(); - node.start(); - checkState(DiscoveryNode.localNode(node.settings())); - checkState(!node.isClosed()); + tmpDir = Files.createTempDirectory("tmp-es"); + } catch (IOException e) { + throw new RuntimeException("Cannot create elasticsearch temporary directory", e); + } - // wait for node to be ready - node.client().admin().cluster().prepareHealth().setWaitForGreenStatus().get(); + tmpDir.toFile().deleteOnExit(); - // delete the indices (should not exist) - DeleteIndexResponse response = node.client().admin().indices().prepareDelete("_all").get(); - checkState(response.isAcknowledged()); - } catch (IOException e) { - throw Throwables.propagate(e); + Settings.Builder settings = Settings.builder() + .put("transport.type", "local") + .put("node.data", true) + .put("cluster.name", nodeName) + .put("node.name", nodeName) + // the two following properties are probably not used because they are + // declared on indices too + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) + // limit the number of threads created (see org.elasticsearch.common.util.concurrent.EsExecutors) + .put("processors", 1) + .put("http.enabled", false) + .put("config.ignore_system_properties", true) + .put("action.auto_create_index", false) + .put("path.home", tmpDir); + node = new Node(settings.build()); + try { + node.start(); + } catch (NodeValidationException e) { + throw new RuntimeException("Cannot start Elasticsearch node", e); } + checkState(!node.isClosed()); + + // wait for node to be ready + node.client().admin().cluster().prepareHealth().setWaitForGreenStatus().get(); + + // delete the indices (should not exist) + DeleteIndexResponse response = node.client().admin().indices().prepareDelete("_all").get(); + checkState(response.isAcknowledged()); } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/IndexCreatorTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/IndexCreatorTest.java index d7b1e669cdbe..02f9a5ca018f 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/IndexCreatorTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/IndexCreatorTest.java @@ -31,6 +31,7 @@ import org.sonar.server.es.metadata.MetadataIndexDefinition; import static org.assertj.core.api.Assertions.assertThat; +import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; import static org.mockito.Matchers.anyString; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.mock; @@ -128,7 +129,7 @@ public void do_not_recreate_index_on_unchanged_definition() throws Exception { IndexType fakeIndexType = new IndexType("fakes", "fake"); String id = "1"; - es.client().prepareIndex(fakeIndexType).setId(id).setSource(new FakeDoc().getFields()).setRefresh(true).get(); + es.client().prepareIndex(fakeIndexType).setId(id).setSource(new FakeDoc().getFields()).setRefreshPolicy(IMMEDIATE).get(); assertThat(es.client().prepareGet(fakeIndexType, id).get().isExists()).isTrue(); // v1 diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/NewIndexTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/NewIndexTest.java index 89482d7871d2..68a8950f7efc 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/NewIndexTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/NewIndexTest.java @@ -63,7 +63,7 @@ public void define_fields() { NewIndex index = new NewIndex("issues"); NewIndex.NewIndexType mapping = index.createType("issue"); mapping.setAttribute("dynamic", "true"); - mapping.setProperty("foo_field", ImmutableMap.of("type", "string"));// ES 5: change to keyword + mapping.setProperty("foo_field", ImmutableMap.of("type", "keyword")); mapping.createBooleanField("boolean_field"); mapping.createByteField("byte_field"); mapping.createDateTimeField("dt_field"); @@ -77,7 +77,7 @@ public void define_fields() { assertThat(mapping).isNotNull(); assertThat(mapping.getAttributes().get("dynamic")).isEqualTo("true"); assertThat(mapping.getProperty("foo_field")).isInstanceOf(Map.class); - assertThat((Map) mapping.getProperty("foo_field")).containsEntry("type", "string");// ES 5: change to keyword + assertThat((Map) mapping.getProperty("foo_field")).containsEntry("type", "keyword"); assertThat((Map) mapping.getProperty("byte_field")).isNotEmpty(); assertThat((Map) mapping.getProperty("double_field")).isNotEmpty(); assertThat((Map) mapping.getProperty("dt_field")).isNotEmpty(); @@ -102,13 +102,13 @@ public void define_string_field() { .build(); Map props = (Map) mapping.getProperty("basic_field"); - assertThat(props.get("type")).isEqualTo("string");// ES 5: change to keyword - assertThat(props.get("index")).isEqualTo("not_analyzed");// ES 5: change to true + assertThat(props.get("type")).isEqualTo("keyword"); + assertThat(props.get("index")).isEqualTo("true"); assertThat(props.get("fields")).isNull(); props = (Map) mapping.getProperty("not_searchable_field"); - assertThat(props.get("type")).isEqualTo("string");// ES 5: change to keyword - assertThat(props.get("index")).isEqualTo("no");// ES 5: change to false + assertThat(props.get("type")).isEqualTo("keyword"); + assertThat(props.get("index")).isEqualTo("false"); assertThat(props.get("fields")).isNull(); props = (Map) mapping.getProperty("all_capabilities_field"); @@ -130,7 +130,7 @@ public void define_nested_field() { assertThat(result.get("type")).isEqualTo("nested"); Map> subProperties = (Map) result.get("properties"); - assertThat(subProperties.get("key").get("type")).isEqualTo("string"); // ES 5: change to "keyword + assertThat(subProperties.get("key").get("type")).isEqualTo("keyword"); assertThat(subProperties.get("value").get("type")).isEqualTo("double"); } @@ -152,7 +152,7 @@ public void use_default_doc_values() { mapping.keywordFieldBuilder("the_doc_value").build(); Map props = (Map) mapping.getProperty("the_doc_value"); - assertThat(props.get("type")).isEqualTo("string");// ES 5: change to keyword + assertThat(props.get("type")).isEqualTo("keyword"); assertThat(props.get("doc_values")).isNull(); } diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/SortingTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/SortingTest.java index 13123a80880f..cf7ee41f4469 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/SortingTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/SortingTest.java @@ -20,19 +20,13 @@ package org.sonar.server.es; import java.util.List; -import org.apache.commons.lang.reflect.FieldUtils; -import org.elasticsearch.action.search.SearchAction; -import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.client.Client; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.sort.SortBuilder; +import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.SortOrder; import org.junit.Test; import org.sonar.server.exceptions.BadRequestException; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.fail; -import static org.mockito.Mockito.mock; public class SortingTest { @@ -58,9 +52,7 @@ public void ascending_sort_on_single_field() throws Exception { Sorting sorting = new Sorting(); sorting.add("updatedAt"); - SearchRequestBuilder request = new SearchRequestBuilder(mock(Client.class), SearchAction.INSTANCE); - sorting.fill(request, "updatedAt", true); - List fields = fields(request); + List fields = sorting.fill("updatedAt", true); assertThat(fields).hasSize(1); expectField(fields.get(0), "updatedAt", "_first", SortOrder.ASC); } @@ -70,9 +62,7 @@ public void descending_sort_on_single_field() throws Exception { Sorting sorting = new Sorting(); sorting.add("updatedAt"); - SearchRequestBuilder request = new SearchRequestBuilder(mock(Client.class), SearchAction.INSTANCE); - sorting.fill(request, "updatedAt", false); - List fields = fields(request); + List fields = sorting.fill("updatedAt", false); assertThat(fields).hasSize(1); expectField(fields.get(0), "updatedAt", "_last", SortOrder.DESC); } @@ -82,9 +72,7 @@ public void ascending_sort_on_single_field_with_missing_in_last_position() throw Sorting sorting = new Sorting(); sorting.add("updatedAt").missingLast(); - SearchRequestBuilder request = new SearchRequestBuilder(mock(Client.class), SearchAction.INSTANCE); - sorting.fill(request, "updatedAt", true); - List fields = fields(request); + List fields = sorting.fill("updatedAt", true); assertThat(fields).hasSize(1); expectField(fields.get(0), "updatedAt", "_last", SortOrder.ASC); } @@ -94,9 +82,7 @@ public void descending_sort_on_single_field_with_missing_in_last_position() thro Sorting sorting = new Sorting(); sorting.add("updatedAt").missingLast(); - SearchRequestBuilder request = new SearchRequestBuilder(mock(Client.class), SearchAction.INSTANCE); - sorting.fill(request, "updatedAt", false); - List fields = fields(request); + List fields = sorting.fill("updatedAt", false); assertThat(fields).hasSize(1); expectField(fields.get(0), "updatedAt", "_first", SortOrder.DESC); } @@ -110,9 +96,7 @@ public void sort_on_multiple_fields() throws Exception { sorting.add("fileLine", "severity").reverse(); sorting.add("fileLine", "key").missingLast(); - SearchRequestBuilder request = new SearchRequestBuilder(mock(Client.class), SearchAction.INSTANCE); - sorting.fill(request, "fileLine", true); - List fields = fields(request); + List fields = sorting.fill("fileLine", true); assertThat(fields).hasSize(4); expectField(fields.get(0), "file", "_first", SortOrder.ASC); expectField(fields.get(1), "line", "_first", SortOrder.ASC); @@ -126,7 +110,7 @@ public void fail_if_unknown_field() { sorting.add("file"); try { - sorting.fill(new SearchRequestBuilder(mock(Client.class), SearchAction.INSTANCE), "unknown", true); + sorting.fill("unknown", true); fail(); } catch (BadRequestException e) { assertThat(e.getMessage()).isEqualTo("Bad sort field: unknown"); @@ -138,32 +122,13 @@ public void default_sorting() throws Exception { Sorting sorting = new Sorting(); sorting.addDefault("file"); - SearchRequestBuilder request = new SearchRequestBuilder(mock(Client.class), SearchAction.INSTANCE); - sorting.fillDefault(request); - List fields = fields(request); + List fields = sorting.fillDefault(); assertThat(fields).hasSize(1); } - private void expectField(SortBuilder field, String expectedField, String expectedMissing, SortOrder expectedSort) throws IllegalAccessException { - assertThat(fieldName(field)).isEqualTo(expectedField); - assertThat(missing(field)).isEqualTo(expectedMissing); - assertThat(order(field)).isEqualTo(expectedSort); - } - - private static List fields(SearchRequestBuilder request) throws IllegalAccessException { - SearchSourceBuilder source = request.internalBuilder(); - return (List) FieldUtils.readField(source, "sorts", true); - } - - private static String fieldName(SortBuilder sortBuilder) throws IllegalAccessException { - return (String) FieldUtils.readField(sortBuilder, "fieldName", true); - } - - private static String missing(SortBuilder sortBuilder) throws IllegalAccessException { - return (String) FieldUtils.readField(sortBuilder, "missing", true); - } - - private static SortOrder order(SortBuilder sortBuilder) throws IllegalAccessException { - return (SortOrder) FieldUtils.readField(sortBuilder, "order", true); + private void expectField(FieldSortBuilder field, String expectedField, String expectedMissing, SortOrder expectedSort) throws IllegalAccessException { + assertThat(field.getFieldName()).isEqualTo(expectedField); + assertThat(field.missing()).isEqualTo(expectedMissing); + assertThat(field.order()).isEqualTo(expectedSort); } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/request/ProxyIndexRequestBuilderTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/request/ProxyIndexRequestBuilderTest.java index 919d4bed0526..8da86b6a0c4b 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/request/ProxyIndexRequestBuilderTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/request/ProxyIndexRequestBuilderTest.java @@ -19,6 +19,7 @@ */ package org.sonar.server.es.request; +import org.elasticsearch.action.DocWriteResponse.Result; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.common.unit.TimeValue; @@ -46,7 +47,7 @@ public void index_with_index_type_and_id() { IndexResponse response = esTester.client().prepareIndex(FakeIndexDefinition.INDEX_TYPE_FAKE) .setSource(FakeIndexDefinition.newDoc(42).getFields()) .get(); - assertThat(response.isCreated()).isTrue(); + assertThat(response.getResult()).isSameAs(Result.CREATED); } @Test @@ -55,7 +56,7 @@ public void trace_logs() { IndexResponse response = esTester.client().prepareIndex(FakeIndexDefinition.INDEX_TYPE_FAKE) .setSource(FakeIndexDefinition.newDoc(42).getFields()) .get(); - assertThat(response.isCreated()).isTrue(); + assertThat(response.getResult()).isSameAs(Result.CREATED); assertThat(logTester.logs(LoggerLevel.TRACE)).hasSize(1); } diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/request/ProxyMultiGetRequestBuilderTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/request/ProxyMultiGetRequestBuilderTest.java index c34b7050e8b5..fefb93bd0d6c 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/request/ProxyMultiGetRequestBuilderTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/request/ProxyMultiGetRequestBuilderTest.java @@ -22,7 +22,7 @@ import org.elasticsearch.action.get.MultiGetRequest; import org.elasticsearch.action.get.MultiGetRequestBuilder; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.search.fetch.source.FetchSourceContext; +import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.junit.Rule; import org.junit.Test; import org.sonar.api.utils.log.LogTester; diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/request/ProxySearchRequestBuilderTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/request/ProxySearchRequestBuilderTest.java index 63a41020a07e..95cf67681763 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/request/ProxySearchRequestBuilderTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/request/ProxySearchRequestBuilderTest.java @@ -63,7 +63,7 @@ public void trace_logs() { @Test public void fail_to_search_bad_query() { try { - esTester.client().prepareSearch(FakeIndexDefinition.INDEX).setQuery("bad query").get(); + esTester.client().prepareSearch("non-existing-index").get(); fail(); } catch (Exception e) { assertThat(e).isInstanceOf(IllegalStateException.class); diff --git a/server/sonar-server/src/test/java/org/sonar/server/test/index/TestIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/test/index/TestIndexerTest.java index 163f5d384e83..4feb494fae3e 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/test/index/TestIndexerTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/test/index/TestIndexerTest.java @@ -185,7 +185,7 @@ private void indexTest(String projectUuid, String fileUuid, String testName, Str .setId(uuid) .setRouting(projectUuid) .setSource(IOUtils.toString(getClass().getResource(format("%s/%s_%s_%s.json", getClass().getSimpleName(), projectUuid, fileUuid, testName)))) - .setRefresh(REFRESH_IMMEDIATE) // ES 5: change to setRefreshPolicy + .setRefreshPolicy(REFRESH_IMMEDIATE) .get(); }