SONAR-9480 fix removal of rules from index when too many failures

SonarSource · Jul 13, 2017 · 7c1a881 · 7c1a881
1 parent f13b6bf
commit 7c1a881
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 11 deletions.
diff --git a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexer.java
@@ -48,7 +48,7 @@
 import static java.util.Arrays.asList;
 import static java.util.Collections.singletonList;
 import static java.util.Objects.requireNonNull;
-import static org.sonar.core.util.stream.MoreCollectors.toArrayList;
+import static org.sonar.core.util.stream.MoreCollectors.toHashSet;
 import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX_TYPE_RULE;
 import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX_TYPE_RULE_EXTENSION;
 
@@ -138,14 +138,14 @@ private IndexingResult doIndexRules(DbSession dbSession, List<EsQueueDto> items)
     BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, new ResiliencyIndexingListener(dbClient, dbSession, items));
     bulkIndexer.start();
 
-    List<RuleKey> ruleKeys = items
+    Set<RuleKey> ruleKeys = items
       .stream()
       .filter(i -> {
         requireNonNull(i.getDocId(), () -> "BUG - " + i + " has not been persisted before indexing");
         return i.getDocType() == EsQueueDto.Type.RULE;
       })
       .map(i -> RuleKey.parse(i.getDocId()))
-      .collect(toArrayList(items.size()));
+      .collect(toHashSet(items.size()));
 
     dbClient.ruleDao().scrollIndexingRulesByKeys(dbSession, ruleKeys,
       // only index requests, no deletion requests.
@@ -158,8 +158,10 @@ private IndexingResult doIndexRules(DbSession dbSession, List<EsQueueDto> items)
 
     // the remaining items reference rows that don't exist in db. They must
     // be deleted from index.
-    ruleKeys.forEach(r -> bulkIndexer.addDeletion(RuleIndexDefinition.INDEX_TYPE_RULE, r.toString(), r.toString()));
-    ruleKeys.forEach(r -> bulkIndexer.addDeletion(RuleIndexDefinition.INDEX_TYPE_RULE_EXTENSION, RuleExtensionDoc.idOf(r, RuleExtensionScope.system()), r.toString()));
+    ruleKeys.forEach(r -> {
+      bulkIndexer.addDeletion(RuleIndexDefinition.INDEX_TYPE_RULE, r.toString(), r.toString());
+      bulkIndexer.addDeletion(RuleIndexDefinition.INDEX_TYPE_RULE_EXTENSION, RuleExtensionDoc.idOf(r, RuleExtensionScope.system()), r.toString());
+    });
 
     return bulkIndexer.stop();
   }
@@ -168,14 +170,14 @@ private IndexingResult doIndexRuleExtensions(DbSession dbSession, List<EsQueueDt
     BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, new ResiliencyIndexingListener(dbClient, dbSession, items));
     bulkIndexer.start();
 
-    List<RuleExtensionId> docIds = items
+    Set<RuleExtensionId> docIds = items
       .stream()
       .filter(i -> {
         requireNonNull(i.getDocId(), () -> "BUG - " + i + " has not been persisted before indexing");
         return i.getDocType() == EsQueueDto.Type.RULE_EXTENSION;
       })
       .map(RuleIndexer::explodeRuleExtensionDocId)
-      .collect(toArrayList(items.size()));
+      .collect(toHashSet(items.size()));
 
     dbClient.ruleDao().scrollIndexingRuleExtensionsByIds(dbSession, docIds,
       // only index requests, no deletion requests.

diff --git a/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexer.java
@@ -45,7 +45,7 @@
 
 import static java.util.Collections.singletonList;
 import static java.util.Objects.requireNonNull;
-import static org.sonar.core.util.stream.MoreCollectors.toArrayList;
+import static org.sonar.core.util.stream.MoreCollectors.toHashSet;
 import static org.sonar.server.user.index.UserIndexDefinition.INDEX_TYPE_USER;
 
 public class UserIndexer implements StartupIndexer, ResilientIndexer {
@@ -113,14 +113,14 @@ public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) {
     if (items.isEmpty()) {
       return new IndexingResult();
     }
-    List<String> logins = items
+    Set<String> logins = items
       .stream()
       .filter(i -> {
         requireNonNull(i.getDocId(), () -> "BUG - " + i + " has not been persisted before indexing");
         return i.getDocType() == EsQueueDto.Type.USER;
       })
       .map(EsQueueDto::getDocId)
-      .collect(toArrayList(items.size()));
+      .collect(toHashSet(items.size()));
 
     ListMultimap<String, String> organizationUuidsByLogin = ArrayListMultimap.create();
     dbClient.organizationMemberDao().selectForUserIndexing(dbSession, logins, organizationUuidsByLogin::put);

diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/RecoveryIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/RecoveryIndexerTest.java
@@ -307,7 +307,7 @@ public void recover_multiple_times_the_same_document() {
       .containsExactlyInAnyOrder(item1.getUuid(), item2.getUuid(), item3.getUuid());
 
     assertThatLogsContain(TRACE, "Elasticsearch recovery - processing 3 USER");
-    assertThatLogsContain(INFO, "Elasticsearch recovery - 3 documents processed [0 failures]");
+    assertThatLogsContain(INFO, "Elasticsearch recovery - 1 documents processed [0 failures]");
   }
 
   private class ProxyUserIndexer extends UserIndexer {