SONAR-9480 fix support of multiple docs to recover with same id

plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/rule/XooRulesDefinition.java

@@ -138,6 +138,12 @@ private void defineRulesXoo(Context context) {
       .setGapDescription("A certified architect will need roughly half an hour to start working on removal of modules, " +
         "then it's about one hour per module.");
 
+    repo
+      .createRule("xoo-template")
+      .setTemplate(true)
+      .setName("Template of rule")
+      .setHtmlDescription("Template to be overridden by custom rules");
+
     repo.done();
 
   }

plugins/sonar-xoo-plugin/src/test/java/org/sonar/xoo/rule/XooRulesDefinitionTest.java

@@ -42,7 +42,7 @@ public void define_xoo_rules() {
     assertThat(repo).isNotNull();
     assertThat(repo.name()).isEqualTo("Xoo");
     assertThat(repo.language()).isEqualTo("xoo");
-    assertThat(repo.rules()).hasSize(17);
+    assertThat(repo.rules()).hasSize(18);
 
     RulesDefinition.Rule rule = repo.rule(OneIssuePerLineSensor.RULE_KEY);
     assertThat(rule.name()).isNotEmpty();

server/sonar-db-dao/src/main/java/org/sonar/db/es/EsQueueDto.java

@@ -93,25 +93,6 @@ public String toString() {
     return sb.toString();
   }
 
-  @Override
-  public boolean equals(Object o) {
-    if (this == o) {
-      return true;
-    }
-    if (!(o instanceof EsQueueDto)) {
-      return false;
-    }
-
-    EsQueueDto that = (EsQueueDto) o;
-
-    return uuid.equals(that.uuid);
-  }
-
-  @Override
-  public int hashCode() {
-    return uuid.hashCode();
-  }
-
   public static EsQueueDto create(Type docType, String docUuid) {
     return new EsQueueDto().setDocType(docType).setDocId(docUuid);
   }

server/sonar-server/src/main/java/org/sonar/server/es/RecoveryIndexer.java

@@ -29,7 +29,7 @@
 import java.util.concurrent.TimeUnit;
 import org.apache.commons.lang.math.RandomUtils;
 import org.sonar.api.Startable;
-import org.sonar.api.config.Settings;
+import org.sonar.api.config.Configuration;
 import org.sonar.api.utils.System2;
 import org.sonar.api.utils.log.Logger;
 import org.sonar.api.utils.log.Loggers;
@@ -63,18 +63,18 @@ public class RecoveryIndexer implements Startable {
       .setNameFormat("RecoveryIndexer-%d")
       .build());
   private final System2 system2;
-  private final Settings settings;
+  private final Configuration config;
   private final DbClient dbClient;
   private final UserIndexer userIndexer;
   private final RuleIndexer ruleIndexer;
   private final ActiveRuleIndexer activeRuleIndexer;
   private final long minAgeInMs;
   private final long loopLimit;
 
-  public RecoveryIndexer(System2 system2, Settings settings, DbClient dbClient,
+  public RecoveryIndexer(System2 system2, Configuration config, DbClient dbClient,
     UserIndexer userIndexer, RuleIndexer ruleIndexer, ActiveRuleIndexer activeRuleIndexer) {
     this.system2 = system2;
-    this.settings = settings;
+    this.config = config;
     this.dbClient = dbClient;
     this.userIndexer = userIndexer;
     this.ruleIndexer = ruleIndexer;
@@ -161,10 +161,7 @@ private static ListMultimap<EsQueueDto.Type, EsQueueDto> groupItemsByType(Collec
   }
 
   private long getSetting(String key, long defaultValue) {
-    long val = settings.getLong(key);
-    if (val <= 0) {
-      val = defaultValue;
-    }
+    long val = config.getLong(key).orElse(defaultValue);
     LOGGER.debug(LOG_PREFIX + "{}={}", key, val);
     return val;
   }

server/sonar-server/src/main/java/org/sonar/server/es/ResiliencyIndexingListener.java

@@ -19,17 +19,15 @@
  */
 package org.sonar.server.es;
 
+import com.google.common.collect.Multimap;
 import java.util.Collection;
-import java.util.Map;
 import java.util.Objects;
 import java.util.function.Function;
 import org.sonar.core.util.stream.MoreCollectors;
 import org.sonar.db.DbClient;
 import org.sonar.db.DbSession;
 import org.sonar.db.es.EsQueueDto;
 
-import static java.util.stream.Collectors.toMap;
-
 /**
  * Clean-up the db table es_queue when documents
  * are successfully indexed so that the recovery
@@ -50,11 +48,12 @@ public ResiliencyIndexingListener(DbClient dbClient, DbSession dbSession, Collec
   @Override
   public void onSuccess(Collection<String> docIds) {
     if (!docIds.isEmpty()) {
-      Map<String, EsQueueDto> itemsById = items.stream().collect(toMap(EsQueueDto::getDocId, Function.identity()));
+      Multimap<String, EsQueueDto> itemsById = items.stream().collect(MoreCollectors.index(EsQueueDto::getDocId, Function.identity()));
 
       Collection<EsQueueDto> itemsToDelete = docIds
         .stream()
         .map(itemsById::get)
+        .flatMap(Collection::stream)
         .filter(Objects::nonNull)
         .collect(MoreCollectors.toArrayList(docIds.size()));
       dbClient.esQueueDao().delete(dbSession, itemsToDelete);

server/sonar-server/src/main/java/org/sonar/server/rule/RuleUpdater.java

@@ -81,8 +81,7 @@ public boolean update(DbSession dbSession, RuleUpdate update, OrganizationDto or
     apply(update, rule, userSession);
     update(dbSession, rule);
     updateParameters(dbSession, organization, update, rule);
-    ruleIndexer.commitAndIndex(dbSession, rule.getKey());
-    ruleIndexer.commitAndIndex(dbSession, organization, rule.getKey());
+    ruleIndexer.commitAndIndex(dbSession, rule.getKey(), organization);
 
     return true;
   }

server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleExtensionDoc.java

@@ -26,7 +26,6 @@
 import org.sonar.api.rule.RuleKey;
 import org.sonar.db.rule.RuleExtensionForIndexingDto;
 import org.sonar.db.rule.RuleForIndexingDto;
-import org.sonar.db.rule.RuleMetadataDto;
 import org.sonar.server.es.BaseDoc;
 
 public class RuleExtensionDoc extends BaseDoc {
@@ -41,7 +40,7 @@ public RuleExtensionDoc() {
 
   @Override
   public String getId() {
-    return getRuleKey() + "|" + getScope().getScope();
+    return idOf(getRuleKey(), getScope());
   }
 
   @Override
@@ -81,13 +80,6 @@ public RuleExtensionDoc setTags(Set<String> tags) {
     return this;
   }
 
-  public static RuleExtensionDoc of(RuleKey key, RuleExtensionScope scope, RuleMetadataDto ruleExtension) {
-    return new RuleExtensionDoc()
-      .setRuleKey(key)
-      .setScope(scope)
-      .setTags(ruleExtension.getTags());
-  }
-
   public static RuleExtensionDoc of(RuleForIndexingDto rule) {
     return new RuleExtensionDoc()
       .setRuleKey(rule.getRuleKey())
@@ -102,6 +94,10 @@ public static RuleExtensionDoc of(RuleExtensionForIndexingDto rule) {
       .setTags(rule.getTagsAsSet());
   }
 
+  public static String idOf(RuleKey ruleKey, RuleExtensionScope scope) {
+    return ruleKey + "|" + scope.getScope();
+  }
+
   @Override
   public String toString() {
     return ReflectionToStringBuilder.toString(this);

server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexer.java

@@ -45,9 +45,10 @@
 import org.sonar.server.es.StartupIndexer;
 
 import static com.google.common.base.Preconditions.checkArgument;
+import static java.util.Arrays.asList;
 import static java.util.Collections.singletonList;
 import static java.util.Objects.requireNonNull;
-import static org.sonar.core.util.stream.MoreCollectors.toHashSet;
+import static org.sonar.core.util.stream.MoreCollectors.toArrayList;
 import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX_TYPE_RULE;
 import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX_TYPE_RULE_EXTENSION;
 
@@ -95,78 +96,70 @@ public void commitAndIndex(DbSession dbSession, RuleKey ruleKey) {
     commitAndIndex(dbSession, singletonList(ruleKey));
   }
 
-  public void commitAndIndex(DbSession dbSession, Collection<RuleKey> ruleDtos) {
-    List<EsQueueDto> items = ruleDtos.stream()
-      .map(key -> EsQueueDto.create(EsQueueDto.Type.RULE, key.toString()))
+  public void commitAndIndex(DbSession dbSession, Collection<RuleKey> ruleKeys) {
+    List<EsQueueDto> items = ruleKeys.stream()
+      .map(RuleIndexer::createQueueDtoForRule)
       .collect(MoreCollectors.toArrayList());
 
     dbClient.esQueueDao().insert(dbSession, items);
     dbSession.commit();
-    postCommit(dbSession, ruleDtos, items);
+    postCommit(dbSession, items);
   }
 
-  public void commitAndIndex(DbSession dbSession, OrganizationDto organizationDto, RuleKey ruleKey) {
-    List<EsQueueDto> items = singletonList(EsQueueDto.create(EsQueueDto.Type.RULE_EXTENSION, ruleKey + "|" + organizationDto.getUuid()));
-
+  /**
+   * Commit a change on a rule and its extension on the given organization
+   */
+  public void commitAndIndex(DbSession dbSession, RuleKey ruleKey, OrganizationDto organization) {
+    List<EsQueueDto> items = asList(createQueueDtoForRule(ruleKey), createQueueDtoForRuleExtension(ruleKey, organization));
     dbClient.esQueueDao().insert(dbSession, items);
     dbSession.commit();
-    postCommit(dbSession, ruleKey, organizationDto, items);
+    postCommit(dbSession, items);
   }
 
   /**
-   * Entry point for Byteman tests. See directory tests/resilience.
-   * The parameter "ruleKeys" is used only by the Byteman script.
+   * This method is used by the Byteman script of integration tests.
    */
-  private void postCommit(DbSession dbSession, Collection<RuleKey> ruleKeys, Collection<EsQueueDto> items) {
-    index(dbSession, items);
-  }
-
-  private void postCommit(DbSession dbSession, RuleKey ruleKeys, OrganizationDto organizationDto, Collection<EsQueueDto> items) {
+  private void postCommit(DbSession dbSession, List<EsQueueDto> items) {
     index(dbSession, items);
   }
 
   @Override
   public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) {
-    if (items.isEmpty()) {
-      return new IndexingResult();
-    }
-
     IndexingResult result = new IndexingResult();
-
-    ListMultimap<EsQueueDto.Type, EsQueueDto> itemsByType = groupItemsByType(items);
-
-    result.add(doIndexRules(dbSession, itemsByType.get(EsQueueDto.Type.RULE)));
-    result.add(doIndexRuleExtensions(dbSession, itemsByType.get(EsQueueDto.Type.RULE_EXTENSION)));
-
+    if (!items.isEmpty()) {
+      ListMultimap<EsQueueDto.Type, EsQueueDto> itemsByType = groupItemsByType(items);
+      result.add(doIndexRules(dbSession, itemsByType.get(EsQueueDto.Type.RULE)));
+      result.add(doIndexRuleExtensions(dbSession, itemsByType.get(EsQueueDto.Type.RULE_EXTENSION)));
+    }
     return result;
   }
 
   private IndexingResult doIndexRules(DbSession dbSession, List<EsQueueDto> items) {
     BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, new ResiliencyIndexingListener(dbClient, dbSession, items));
     bulkIndexer.start();
 
-    Set<RuleKey> rules = items
+    List<RuleKey> ruleKeys = items
       .stream()
       .filter(i -> {
         requireNonNull(i.getDocId(), () -> "BUG - " + i + " has not been persisted before indexing");
         return i.getDocType() == EsQueueDto.Type.RULE;
       })
       .map(i -> RuleKey.parse(i.getDocId()))
-      .collect(toHashSet(items.size()));
+      .collect(toArrayList(items.size()));
 
-    dbClient.ruleDao().scrollIndexingRulesByKeys(dbSession, rules,
+    dbClient.ruleDao().scrollIndexingRulesByKeys(dbSession, ruleKeys,
       // only index requests, no deletion requests.
       // Deactivated users are not deleted but updated.
       r -> {
-        rules.remove(r.getRuleKey());
         bulkIndexer.add(newRuleDocIndexRequest(r));
         bulkIndexer.add(newRuleExtensionDocIndexRequest(r));
+        ruleKeys.remove(r.getRuleKey());
       });
 
     // the remaining items reference rows that don't exist in db. They must
     // be deleted from index.
-    rules.forEach(r -> bulkIndexer.addDeletion(RuleIndexDefinition.INDEX_TYPE_RULE, r.toString()));
-    rules.forEach(r -> bulkIndexer.addDeletion(RuleIndexDefinition.INDEX_TYPE_RULE_EXTENSION, r.toString()));
+    ruleKeys.forEach(r -> bulkIndexer.addDeletion(RuleIndexDefinition.INDEX_TYPE_RULE, r.toString(), r.toString()));
+    ruleKeys.forEach(r -> bulkIndexer.addDeletion(RuleIndexDefinition.INDEX_TYPE_RULE_EXTENSION, RuleExtensionDoc.idOf(r, RuleExtensionScope.system()), r.toString()));
 
     return bulkIndexer.stop();
   }
@@ -175,26 +168,30 @@ private IndexingResult doIndexRuleExtensions(DbSession dbSession, List<EsQueueDt
     BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, new ResiliencyIndexingListener(dbClient, dbSession, items));
     bulkIndexer.start();
 
-    Set<RuleExtensionId> docIds = items
+    List<RuleExtensionId> docIds = items
       .stream()
       .filter(i -> {
         requireNonNull(i.getDocId(), () -> "BUG - " + i + " has not been persisted before indexing");
         return i.getDocType() == EsQueueDto.Type.RULE_EXTENSION;
       })
       .map(RuleIndexer::explodeRuleExtensionDocId)
-      .collect(toHashSet(items.size()));
+      .collect(toArrayList(items.size()));
 
     dbClient.ruleDao().scrollIndexingRuleExtensionsByIds(dbSession, docIds,
       // only index requests, no deletion requests.
       // Deactivated users are not deleted but updated.
       r -> {
-        docIds.remove(new RuleExtensionId(r.getOrganizationUuid(), r.getPluginName(), r.getPluginRuleKey()));
+        RuleExtensionId docId = new RuleExtensionId(r.getOrganizationUuid(), r.getPluginName(), r.getPluginRuleKey());
+        docIds.remove(docId);
         bulkIndexer.add(newRuleExtensionDocIndexRequest(r));
       });
 
     // the remaining items reference rows that don't exist in db. They must
     // be deleted from index.
-    docIds.forEach(r -> bulkIndexer.addDeletion(RuleIndexDefinition.INDEX_TYPE_RULE_EXTENSION, r.getId()));
+    docIds.forEach(docId -> {
+      RuleKey ruleKey = RuleKey.of(docId.getRepositoryName(), docId.getRuleKey());
+      bulkIndexer.addDeletion(RuleIndexDefinition.INDEX_TYPE_RULE_EXTENSION, docId.getId(), ruleKey.toString());
+    });
 
     return bulkIndexer.stop();
   }
@@ -239,4 +236,14 @@ private static RuleExtensionId explodeRuleExtensionDocId(EsQueueDto esQueueDto)
     checkArgument(esQueueDto.getDocType() == EsQueueDto.Type.RULE_EXTENSION);
     return new RuleExtensionId(esQueueDto.getDocId());
   }
+
+  private static EsQueueDto createQueueDtoForRule(RuleKey ruleKey) {
+    return EsQueueDto.create(EsQueueDto.Type.RULE, ruleKey.toString(), null, ruleKey.toString());
+  }
+
+  private static EsQueueDto createQueueDtoForRuleExtension(RuleKey ruleKey, OrganizationDto organization) {
+    String docId = RuleExtensionDoc.idOf(ruleKey, RuleExtensionScope.organization(organization));
+    return EsQueueDto.create(EsQueueDto.Type.RULE_EXTENSION, docId, null, ruleKey.toString());
+  }
+
 }

server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexer.java

@@ -38,14 +38,14 @@
 import org.sonar.server.es.EsClient;
 import org.sonar.server.es.IndexType;
 import org.sonar.server.es.IndexingListener;
+import org.sonar.server.es.IndexingResult;
 import org.sonar.server.es.ResiliencyIndexingListener;
 import org.sonar.server.es.ResilientIndexer;
-import org.sonar.server.es.IndexingResult;
 import org.sonar.server.es.StartupIndexer;
 
 import static java.util.Collections.singletonList;
 import static java.util.Objects.requireNonNull;
-import static org.sonar.core.util.stream.MoreCollectors.toHashSet;
+import static org.sonar.core.util.stream.MoreCollectors.toArrayList;
 import static org.sonar.server.user.index.UserIndexDefinition.INDEX_TYPE_USER;
 
 public class UserIndexer implements StartupIndexer, ResilientIndexer {
@@ -113,14 +113,14 @@ public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) {
     if (items.isEmpty()) {
       return new IndexingResult();
     }
-    Set<String> logins = items
+    List<String> logins = items
       .stream()
       .filter(i -> {
         requireNonNull(i.getDocId(), () -> "BUG - " + i + " has not been persisted before indexing");
         return i.getDocType() == EsQueueDto.Type.USER;
       })
       .map(EsQueueDto::getDocId)
-      .collect(toHashSet(items.size()));
+      .collect(toArrayList(items.size()));
 
     ListMultimap<String, String> organizationUuidsByLogin = ArrayListMultimap.create();
     dbClient.organizationMemberDao().selectForUserIndexing(dbSession, logins, organizationUuidsByLogin::put);