SONAR-7330 Add rule index definition

server/sonar-server/src/main/java/org/sonar/server/es/BaseIndex.java

@@ -23,6 +23,11 @@
 
 @ServerSide
 public abstract class BaseIndex {
+
+  public static final String SORT_SUFFIX = "sort";
+  public static final String SEARCH_WORDS_SUFFIX = "words";
+  public static final String SEARCH_PARTIAL_SUFFIX = "grams";
+
   private final EsClient client;
 
   public BaseIndex(EsClient client) {

server/sonar-server/src/main/java/org/sonar/server/es/EsUtils.java

@@ -21,17 +21,6 @@
 
 import com.google.common.base.Function;
 import com.google.common.collect.Lists;
-import org.elasticsearch.action.search.SearchScrollRequestBuilder;
-import org.elasticsearch.common.joda.time.format.ISODateTimeFormat;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.search.SearchHit;
-import org.elasticsearch.search.SearchHits;
-import org.elasticsearch.search.aggregations.bucket.terms.Terms;
-import org.sonar.server.search.BaseDoc;
-
-import javax.annotation.CheckForNull;
-import javax.annotation.Nullable;
-
 import java.util.ArrayDeque;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -42,6 +31,15 @@
 import java.util.Map;
 import java.util.NoSuchElementException;
 import java.util.Queue;
+import javax.annotation.CheckForNull;
+import javax.annotation.Nullable;
+import org.elasticsearch.action.search.SearchScrollRequestBuilder;
+import org.elasticsearch.common.joda.time.format.ISODateTimeFormat;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchHits;
+import org.elasticsearch.search.aggregations.bucket.terms.Terms;
+import org.sonar.server.search.BaseDoc;
 
 public class EsUtils {
 
@@ -121,4 +119,33 @@ public void remove() {
       }
     };
   }
+
+  public static <ID> Iterator<ID> scrollIds(final EsClient esClient, final String scrollId, final Function<String, ID> idConverter) {
+    return new Iterator<ID>() {
+      private final Queue<SearchHit> hits = new ArrayDeque<>();
+
+      @Override
+      public boolean hasNext() {
+        if (hits.isEmpty()) {
+          SearchScrollRequestBuilder esRequest = esClient.prepareSearchScroll(scrollId)
+            .setScroll(TimeValue.timeValueMinutes(SCROLL_TIME_IN_MINUTES));
+          Collections.addAll(hits, esRequest.get().getHits().getHits());
+        }
+        return !hits.isEmpty();
+      }
+
+      @Override
+      public ID next() {
+        if (!hasNext()) {
+          throw new NoSuchElementException();
+        }
+        return idConverter.apply(hits.poll().getId());
+      }
+
+      @Override
+      public void remove() {
+        throw new UnsupportedOperationException("Cannot remove item when scrolling");
+      }
+    };
+  }
 }

server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java

@@ -54,7 +54,7 @@ private NewIndexType(String typeName) {
       // defaults
       attributes.put("dynamic", false);
       attributes.put("_all", ImmutableSortedMap.of("enabled", false));
-
+      attributes.put("_source", ImmutableSortedMap.of("enabled", true));
       attributes.put("properties", properties);
     }
 
@@ -78,6 +78,11 @@ public NewIndexType setProperty(String key, Object value) {
       return this;
     }
 
+    public NewIndexType setEnableSource(boolean enableSource) {
+      attributes.put("_source", ImmutableSortedMap.of("enabled", enableSource));
+      return this;
+    }
+
     public StringFieldBuilder stringFieldBuilder(String fieldName) {
       return new StringFieldBuilder(this, fieldName);
     }
@@ -250,7 +255,13 @@ public StringFieldBuilder disableSearch() {
     public void build() {
       validate();
       Map<String, Object> hash = new TreeMap<>();
-      if (!subFields.isEmpty()) {
+      if (subFields.isEmpty()) {
+        hash.putAll(ImmutableMap.of(
+          "type", "string",
+          "index", disableSearch ? "no" : "not_analyzed",
+          "omit_norms", "true",
+          "doc_values", docValues));
+      } else {
         hash.put("type", "multi_field");
         Map<String, Object> multiFields = new TreeMap<>(subFields);
         multiFields.put(fieldName, ImmutableMap.of(
@@ -259,12 +270,6 @@ public void build() {
           "omit_norms", "true",
           "doc_values", docValues));
         hash.put("fields", multiFields);
-      } else {
-        hash.putAll(ImmutableMap.of(
-          "type", "string",
-          "index", disableSearch ? "no" : "not_analyzed",
-          "omit_norms", "true",
-          "doc_values", docValues));
       }
 
       indexType.setProperty(fieldName, hash);

server/sonar-server/src/main/java/org/sonar/server/es/SearchIdResult.java

@@ -0,0 +1,66 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2016 SonarSource SA
+ * mailto:contact AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.es;
+
+import com.google.common.base.Function;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.commons.lang.builder.ReflectionToStringBuilder;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchHits;
+
+public class SearchIdResult<ID> {
+
+  private final List<ID> ids;
+  private final Facets facets;
+  private final long total;
+
+  public SearchIdResult(SearchResponse response, Function<String, ID> converter) {
+    this.facets = new Facets(response);
+    this.total = response.getHits().totalHits();
+    this.ids = convertToIds(response.getHits(), converter);
+  }
+
+  public List<ID> getIds() {
+    return ids;
+  }
+
+  public long getTotal() {
+    return total;
+  }
+
+  public Facets getFacets() {
+    return this.facets;
+  }
+
+  @Override
+  public String toString() {
+    return ReflectionToStringBuilder.toString(this);
+  }
+
+  public static <ID> List<ID> convertToIds(SearchHits hits, Function<String, ID> converter) {
+    List<ID> docs = new ArrayList<>();
+    for (SearchHit hit : hits.getHits()) {
+      docs.add(converter.apply(hit.getId()));
+    }
+    return docs;
+  }
+}

server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java

@@ -76,12 +76,12 @@
 import org.sonar.server.es.Sorting;
 import org.sonar.server.exceptions.NotFoundException;
 import org.sonar.server.issue.IssueQuery;
-import org.sonarqube.ws.client.issue.IssueFilterParameters;
-import org.sonar.server.rule.index.RuleNormalizer;
+import org.sonar.server.rule.index.RuleIndexDefinition;
 import org.sonar.server.search.IndexDefinition;
 import org.sonar.server.search.StickyFacetBuilder;
 import org.sonar.server.user.UserSession;
 import org.sonar.server.view.index.ViewIndexDefinition;
+import org.sonarqube.ws.client.issue.IssueFilterParameters;
 
 import static com.google.common.collect.Lists.newArrayList;
 
@@ -652,7 +652,7 @@ public List<String> listTags(IssueQuery query, @Nullable String textQuery, int m
       issueTags.include(String.format(SUBSTRING_MATCH_REGEXP, textQuery));
     }
     TermsBuilder ruleTags = AggregationBuilders.terms(tagsOnRulesSubAggregation)
-      .field(RuleNormalizer.RuleField.ALL_TAGS.field())
+      .field(RuleIndexDefinition.FIELD_RULE_ALL_TAGS)
       .size(maxNumberOfTags)
       .order(Terms.Order.term(true))
       .minDocCount(1L);

server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java

@@ -106,7 +106,7 @@ public void deleteProject(String uuid, boolean refresh) {
     bulk.stop();
   }
 
-  BulkIndexer createBulkIndexer(boolean large) {
+  private BulkIndexer createBulkIndexer(boolean large) {
     BulkIndexer bulk = new BulkIndexer(esClient, IssueIndexDefinition.INDEX);
     bulk.setLarge(large);
     return bulk;

server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel1.java

@@ -48,6 +48,7 @@
 import org.sonar.server.ruby.PlatformRackBridge;
 import org.sonar.server.rule.db.RuleDao;
 import org.sonar.server.rule.index.RuleIndex;
+import org.sonar.server.rule.index.RuleIndex2;
 import org.sonar.server.rule.index.RuleNormalizer;
 import org.sonar.server.search.EsSearchModule;
 import org.sonar.server.search.IndexQueue;
@@ -108,6 +109,7 @@ public void configureLevel() {
       ActiveRuleDao.class,
 
       // rules/qprofiles
+      RuleIndex2.class,
       RuleNormalizer.class,
       ActiveRuleNormalizer.class,
       RuleIndex.class,

server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java

@@ -256,6 +256,8 @@
 import org.sonar.server.rule.RuleRepositories;
 import org.sonar.server.rule.RuleService;
 import org.sonar.server.rule.RuleUpdater;
+import org.sonar.server.rule.index.RuleIndexDefinition;
+import org.sonar.server.rule.index.RuleIndexer;
 import org.sonar.server.rule.ws.ActiveRuleCompleter;
 import org.sonar.server.rule.ws.RepositoriesAction;
 import org.sonar.server.rule.ws.RuleMapper;
@@ -430,6 +432,8 @@ protected void configureLevel() {
       RubyQProfileActivityService.class,
 
       // rule
+      RuleIndexDefinition.class,
+      RuleIndexer.class,
       AnnotationRuleParser.class,
       XMLRuleParser.class,
       DefaultRuleFinder.class,

server/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndex.java

@@ -21,6 +21,11 @@
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Multimap;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchType;
@@ -36,19 +41,13 @@
 import org.sonar.db.qualityprofile.ActiveRuleDto;
 import org.sonar.db.qualityprofile.ActiveRuleKey;
 import org.sonar.server.qualityprofile.ActiveRule;
-import org.sonar.server.rule.index.RuleNormalizer;
+import org.sonar.server.rule.index.RuleIndexDefinition;
 import org.sonar.server.search.BaseIndex;
 import org.sonar.server.search.FacetValue;
 import org.sonar.server.search.IndexDefinition;
 import org.sonar.server.search.IndexField;
 import org.sonar.server.search.SearchClient;
 
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
 public class ActiveRuleIndex extends BaseIndex<ActiveRule, ActiveRuleDto, ActiveRuleKey> {
 
   public static final String COUNT_ACTIVE_RULES = "countActiveRules";
@@ -139,7 +138,7 @@ public Iterator<ActiveRule> findByProfile(String key) {
       .setQuery(QueryBuilders.filteredQuery(QueryBuilders.matchAllQuery(), FilterBuilders.boolFilter()
         .must(FilterBuilders.termFilter(ActiveRuleNormalizer.ActiveRuleField.PROFILE_KEY.field(), key))
         .mustNot(FilterBuilders.hasParentFilter(this.getParentType(),
-          FilterBuilders.termFilter(RuleNormalizer.RuleField.STATUS.field(), RuleStatus.REMOVED.name())))))
+          FilterBuilders.termFilter(RuleIndexDefinition.FIELD_STATUS, RuleStatus.REMOVED.name())))))
       .setRouting(key);
 
     SearchResponse response = request.get();
@@ -154,14 +153,14 @@ public Long countByQualityProfileKey(String key) {
     return countByField(ActiveRuleNormalizer.ActiveRuleField.PROFILE_KEY,
       FilterBuilders.hasParentFilter(IndexDefinition.RULE.getIndexType(),
         FilterBuilders.notFilter(
-          FilterBuilders.termFilter(RuleNormalizer.RuleField.STATUS.field(), "REMOVED")))).get(key);
+          FilterBuilders.termFilter(RuleIndexDefinition.FIELD_STATUS, "REMOVED")))).get(key);
   }
 
   public Map<String, Long> countAllByQualityProfileKey() {
     return countByField(ActiveRuleNormalizer.ActiveRuleField.PROFILE_KEY,
       FilterBuilders.hasParentFilter(IndexDefinition.RULE.getIndexType(),
         FilterBuilders.notFilter(
-          FilterBuilders.termFilter(RuleNormalizer.RuleField.STATUS.field(), "REMOVED"))));
+          FilterBuilders.termFilter(RuleIndexDefinition.FIELD_STATUS, "REMOVED"))));
   }
 
   public Multimap<String, FacetValue> getStatsByProfileKey(String key) {
@@ -174,7 +173,7 @@ public Map<String, Multimap<String, FacetValue>> getStatsByProfileKeys(List<Stri
         QueryBuilders.termsQuery(ActiveRuleNormalizer.ActiveRuleField.PROFILE_KEY.field(), keys),
         FilterBuilders.boolFilter()
           .mustNot(FilterBuilders.hasParentFilter(this.getParentType(),
-            FilterBuilders.termFilter(RuleNormalizer.RuleField.STATUS.field(), RuleStatus.REMOVED.name())))))
+            FilterBuilders.termFilter(RuleIndexDefinition.FIELD_STATUS, RuleStatus.REMOVED.name())))))
       .addAggregation(AggregationBuilders.terms(ActiveRuleNormalizer.ActiveRuleField.PROFILE_KEY.field())
         .field(ActiveRuleNormalizer.ActiveRuleField.PROFILE_KEY.field()).size(0)
         .subAggregation(AggregationBuilders.terms(ActiveRuleNormalizer.ActiveRuleField.INHERITANCE.field())