diff --git a/server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevelStartup.java b/server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevelStartup.java index 588ae28ead92..c933b8f91221 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevelStartup.java +++ b/server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevelStartup.java @@ -25,8 +25,12 @@ import org.sonar.server.platform.ServerLifecycleNotifier; import org.sonar.server.platform.web.RegisterServletFilters; import org.sonar.server.qualitygate.RegisterQualityGates; +import org.sonar.server.qualityprofile.CachingDefinedQProfileCreationImpl; +import org.sonar.server.qualityprofile.CachingRuleActivator; +import org.sonar.server.qualityprofile.CachingRuleActivatorContextFactory; import org.sonar.server.qualityprofile.DefinedQProfileInsertImpl; import org.sonar.server.qualityprofile.DefinedQProfileLoader; +import org.sonar.server.qualityprofile.MassRegisterQualityProfiles; import org.sonar.server.qualityprofile.RegisterQualityProfiles; import org.sonar.server.rule.RegisterRules; import org.sonar.server.startup.DeleteOldAnalysisReportsFromFs; @@ -57,6 +61,10 @@ protected void configureLevel() { add(DefinedQProfileLoader.class); addIfStartupLeader( DefinedQProfileInsertImpl.class, + MassRegisterQualityProfiles.class, + CachingRuleActivatorContextFactory.class, + CachingRuleActivator.class, + CachingDefinedQProfileCreationImpl.class, RegisterQualityProfiles.class, RegisterPermissionTemplates.class, RenameDeprecatedPropertyKeys.class, diff --git a/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/CachingDefinedQProfileCreation.java b/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/CachingDefinedQProfileCreation.java new file mode 100644 index 000000000000..4f973d6411bb --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/CachingDefinedQProfileCreation.java @@ -0,0 +1,26 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.qualityprofile; + +/** + * Marker interface of any implementation of {@link DefinedQProfileCreation} which supports caching. + */ +public interface CachingDefinedQProfileCreation extends DefinedQProfileCreation { +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/CachingDefinedQProfileCreationImpl.java b/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/CachingDefinedQProfileCreationImpl.java new file mode 100644 index 000000000000..1d4e7550871a --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/CachingDefinedQProfileCreationImpl.java @@ -0,0 +1,28 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.qualityprofile; + +import org.sonar.db.DbClient; + +public class CachingDefinedQProfileCreationImpl extends DefinedQProfileCreationImpl implements CachingDefinedQProfileCreation { + public CachingDefinedQProfileCreationImpl(DbClient dbClient, QProfileFactory profileFactory, CachingRuleActivator ruleActivator) { + super(dbClient, profileFactory, ruleActivator); + } +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/CachingRuleActivator.java b/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/CachingRuleActivator.java new file mode 100644 index 000000000000..452ef1fb31f9 --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/CachingRuleActivator.java @@ -0,0 +1,55 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.qualityprofile; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import java.util.List; +import org.sonar.api.utils.System2; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.db.qualityprofile.QualityProfileDto; +import org.sonar.server.qualityprofile.index.ActiveRuleIndexer; +import org.sonar.server.rule.index.RuleIndex; +import org.sonar.server.user.UserSession; +import org.sonar.server.util.TypeValidations; + +public class CachingRuleActivator extends RuleActivator { + private final Cache> childrenByParentKey = CacheBuilder.newBuilder() + .maximumSize(10_000) + .build(); + + public CachingRuleActivator(System2 system2, DbClient db, RuleIndex ruleIndex, CachingRuleActivatorContextFactory contextFactory, TypeValidations typeValidations, + ActiveRuleIndexer activeRuleIndexer, UserSession userSession) { + super(system2, db, ruleIndex, contextFactory, typeValidations, activeRuleIndexer, userSession); + } + + @Override + protected List getChildren(DbSession session, String qualityProfileKey) { + List res = childrenByParentKey.getIfPresent(qualityProfileKey); + if (res != null) { + return res; + } + res = super.getChildren(session, qualityProfileKey); + childrenByParentKey.put(qualityProfileKey, res); + return res; + } + +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/CachingRuleActivatorContextFactory.java b/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/CachingRuleActivatorContextFactory.java new file mode 100644 index 000000000000..befd97398bbb --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/CachingRuleActivatorContextFactory.java @@ -0,0 +1,83 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.qualityprofile; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ExecutionException; +import org.picocontainer.Startable; +import org.sonar.api.rule.RuleKey; +import org.sonar.core.util.stream.MoreCollectors; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.db.qualityprofile.ActiveRuleDto; +import org.sonar.db.qualityprofile.ActiveRuleKey; +import org.sonar.db.rule.RuleDefinitionDto; + +public class CachingRuleActivatorContextFactory extends RuleActivatorContextFactory implements Startable { + private final DbClient dbClient; + private final Map rulesByRuleKey = new HashMap<>(); + private final Cache> childrenByParentKey = CacheBuilder.newBuilder() + .maximumSize(10) + .build(); + + public CachingRuleActivatorContextFactory(DbClient db) { + super(db); + this.dbClient = db; + } + + @Override + public void start() { + try (DbSession dbSession = dbClient.openSession(false)) { + dbClient.ruleDao().selectAllDefinitions(dbSession).forEach(rule -> rulesByRuleKey.put(rule.getKey(), rule)); + } + } + + @Override + public void stop() { + // nothing to do + } + + @Override + Optional getRule(DbSession dbSession, RuleKey ruleKey) { + return Optional.ofNullable(rulesByRuleKey.get(ruleKey)); + } + + @Override + Optional getActiveRule(DbSession session, ActiveRuleKey key) { + try { + String profileKey = key.qProfile(); + Map profileActiveRulesByRuleKey = childrenByParentKey.get( + profileKey, + () -> loadActiveRulesOfQualityProfile(session, profileKey)); + return Optional.ofNullable(profileActiveRulesByRuleKey.get(key.ruleKey())); + } catch (ExecutionException e) { + throw new IllegalStateException(e.getCause()); + } + } + + private Map loadActiveRulesOfQualityProfile(DbSession session, String profileKey) { + return dbClient.activeRuleDao().selectByProfileKey(session, profileKey).stream() + .collect(MoreCollectors.uniqueIndex(dto -> dto.getKey().ruleKey())); + } +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/DefinedQProfileInsertImpl.java b/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/DefinedQProfileInsertImpl.java index cf0dfad78b30..e52eda88b061 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/DefinedQProfileInsertImpl.java +++ b/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/DefinedQProfileInsertImpl.java @@ -20,10 +20,18 @@ package org.sonar.server.qualityprofile; import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; +import java.util.Collections; import java.util.Date; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; import javax.annotation.CheckForNull; import javax.annotation.Nullable; import org.sonar.api.rule.RuleKey; @@ -47,13 +55,14 @@ import static com.google.common.base.MoreObjects.firstNonNull; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.Lists.newArrayList; +import static java.util.Objects.requireNonNull; public class DefinedQProfileInsertImpl implements DefinedQProfileInsert { private final DbClient dbClient; private final System2 system2; private final UuidFactory uuidFactory; private final TypeValidations typeValidations; - private RegisterQualityProfiles.RuleRepository ruleRepository; + private RuleRepository ruleRepository; public DefinedQProfileInsertImpl(DbClient dbClient, System2 system2, UuidFactory uuidFactory, TypeValidations typeValidations) { this.dbClient = dbClient; @@ -68,10 +77,6 @@ public void create(DbSession session, DefinedQProfile definedQProfile, Organizat checkArgument(definedQProfile.getParentQProfileName() == null, "Inheritance of Quality Profiles is not supported yet"); - // Optional.ofNullable(definedQProfile.getParentQProfileName()) - // .map(parentQProfileName -> dbClient.qualityProfileDao().selectByNameAndLanguage(organization, parentQProfileName.getName(), - // parentQProfileName.getLanguage(), session)) - // .map(parentQualityProfileDto -> dbClient.activeRuleDao().selectByRuleIds()); Date now = new Date(system2.now()); QualityProfileDto profileDto = insertQualityProfile(session, definedQProfile, organization, now); @@ -90,7 +95,7 @@ public void create(DbSession session, DefinedQProfile definedQProfile, Organizat private void initRuleRepository(DbSession session) { if (ruleRepository == null) { - ruleRepository = new RegisterQualityProfiles.RuleRepository(dbClient, session); + ruleRepository = new RuleRepository(dbClient, session); } } @@ -111,15 +116,7 @@ private ActiveRuleChange insertActiveRule(DbSession session, QualityProfileDto p .orElseThrow(() -> new IllegalStateException("RuleDefinition not found for key " + ruleKey)); ActiveRuleDto dto = ActiveRuleDto.createFor(profileDto, ruleDefinitionDto); - dto.setSeverity( - firstNonNull( - activeRule.getSeverity().name(), - // context.parentSeverity(), - ruleDefinitionDto.getSeverityString())); - // ActiveRule.Inheritance inheritance = activeRule.getInheritance(); - // if (inheritance != null) { - // activeRule.setInheritance(inheritance.name()); - // } + dto.setSeverity(firstNonNull(activeRule.getSeverity().name(), ruleDefinitionDto.getSeverityString())); dto.setUpdatedAt(now); dto.setCreatedAt(now); dbClient.activeRuleDao().insert(session, dto); @@ -173,4 +170,37 @@ private void insertTemplate(DbSession session, DefinedQProfile qualityProfile, O LoadedTemplateDto template = new LoadedTemplateDto(organization.getUuid(), qualityProfile.getLoadedTemplateType()); dbClient.loadedTemplateDao().insert(template, session); } + + public static class RuleRepository { + private final Map ruleDefinitions; + private final Map> ruleParams; + + private RuleRepository(DbClient dbClient, DbSession session) { + this.ruleDefinitions = dbClient.ruleDao().selectAllDefinitions(session) + .stream() + .collect(Collectors.toMap(RuleDefinitionDto::getKey, Function.identity())); + Map ruleIdsByKey = ruleDefinitions.values() + .stream() + .collect(MoreCollectors.uniqueIndex(RuleDefinitionDto::getId, RuleDefinitionDto::getKey)); + this.ruleParams = new HashMap<>(ruleIdsByKey.size()); + dbClient.ruleDao().selectRuleParamsByRuleKeys(session, ruleDefinitions.keySet()) + .forEach(ruleParam -> ruleParams.compute( + ruleIdsByKey.get(ruleParam.getRuleId()), + (key, value) -> { + if (value == null) { + return ImmutableSet.of(ruleParam); + } + return ImmutableSet.copyOf(Sets.union(value, Collections.singleton(ruleParam))); + })); + } + + Optional getDefinition(RuleKey ruleKey) { + return Optional.ofNullable(ruleDefinitions.get(requireNonNull(ruleKey, "RuleKey can't be null"))); + } + + Set getRuleParams(RuleKey ruleKey) { + Set res = ruleParams.get(requireNonNull(ruleKey, "RuleKey can't be null")); + return res == null ? Collections.emptySet() : res; + } + } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/MassRegisterQualityProfiles.java b/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/MassRegisterQualityProfiles.java new file mode 100644 index 000000000000..6e2a2e769537 --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/MassRegisterQualityProfiles.java @@ -0,0 +1,117 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.qualityprofile; + +import java.util.ArrayList; +import java.util.List; +import org.sonar.api.config.Settings; +import org.sonar.api.server.ServerSide; +import org.sonar.api.utils.log.Logger; +import org.sonar.api.utils.log.Loggers; +import org.sonar.api.utils.log.Profiler; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.db.Pagination; +import org.sonar.db.organization.OrganizationDto; +import org.sonar.server.qualityprofile.index.ActiveRuleIndexer; + +import static java.lang.String.format; +import static org.sonar.db.Pagination.forPage; + +/** + * When a property "sonar.qp.massInserts" contains true, this will perform the synchronization of Quality Profiles + * in DB for every organization with the Quality Profiles defined by plugins before and in place of + * {@link RegisterQualityProfiles}. + * + * This implementation is more efficient than the one of {@link RegisterQualityProfiles} but has a strong limitation: + * hierarchies of quality profiles are not supported and an exception will be raised if any child quality profiles + * is encountered + */ +@ServerSide +public class MassRegisterQualityProfiles { + + private static final Logger LOGGER = Loggers.get(MassRegisterQualityProfiles.class); + private static final Pagination PROCESSED_ORGANIZATIONS_BATCH_SIZE = forPage(1).andSize(2000); + + private final Settings settings; + private final DefinedQProfileRepository definedQProfileRepository; + private final DbClient dbClient; + private final ActiveRuleIndexer activeRuleIndexer; + private final DefinedQProfileInsert definedQProfileInsert; + + public MassRegisterQualityProfiles(Settings settings, DefinedQProfileRepository definedQProfileRepository, + DbClient dbClient, DefinedQProfileInsert definedQProfileInsert, ActiveRuleIndexer activeRuleIndexer) { + this.settings = settings; + this.definedQProfileRepository = definedQProfileRepository; + this.dbClient = dbClient; + this.activeRuleIndexer = activeRuleIndexer; + this.definedQProfileInsert = definedQProfileInsert; + } + + public void start() { + if (!settings.getBoolean("sonar.qp.massInserts")) { + return; + } + + Profiler profiler = Profiler.create(Loggers.get(getClass())).startInfo("Mass Register quality profiles"); + if (definedQProfileRepository.getQProfilesByLanguage().isEmpty()) { + return; + } + + try (DbSession session = dbClient.openSession(false)) { + List changes = new ArrayList<>(); + definedQProfileRepository.getQProfilesByLanguage() + .forEach((key, value) -> registerPerLanguage(session, value, changes)); + activeRuleIndexer.index(changes); + profiler.stopDebug(); + } + } + + private void registerPerLanguage(DbSession session, List qualityProfiles, List changes) { + qualityProfiles.forEach(qp -> registerPerQualityProfile(session, qp, changes)); + } + + private void registerPerQualityProfile(DbSession session, DefinedQProfile qualityProfile, List changes) { + LOGGER.info("Register profile {}", qualityProfile.getQProfileName()); + + Profiler profiler = Profiler.create(Loggers.get(getClass())); + List organizationDtos; + while (!(organizationDtos = getOrganizationsWithoutQP(session, qualityProfile)).isEmpty()) { + organizationDtos.forEach(organization -> registerPerQualityProfileAndOrganization(session, qualityProfile, organization, changes, profiler)); + } + } + + private List getOrganizationsWithoutQP(DbSession session, DefinedQProfile qualityProfile) { + return dbClient.organizationDao().selectOrganizationsWithoutLoadedTemplate(session, + qualityProfile.getLoadedTemplateType(), PROCESSED_ORGANIZATIONS_BATCH_SIZE); + } + + private void registerPerQualityProfileAndOrganization(DbSession session, + DefinedQProfile definedQProfile, OrganizationDto organization, List changes, Profiler profiler) { + profiler.start(); + + definedQProfileInsert.create(session, definedQProfile, organization, changes); + + session.commit(); + + profiler.stopDebug(format("Register profile %s for organization %s", definedQProfile.getQProfileName(), organization.getKey())); + } + +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/RegisterQualityProfiles.java b/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/RegisterQualityProfiles.java index 95601f99af64..c003b4d5ba2a 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/RegisterQualityProfiles.java +++ b/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/RegisterQualityProfiles.java @@ -19,37 +19,18 @@ */ package org.sonar.server.qualityprofile; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Sets; import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.function.Function; -import java.util.stream.Collectors; -import org.sonar.api.rule.RuleKey; import org.sonar.api.server.ServerSide; import org.sonar.api.utils.log.Logger; import org.sonar.api.utils.log.Loggers; import org.sonar.api.utils.log.Profiler; -import org.sonar.core.util.stream.MoreCollectors; import org.sonar.db.DbClient; import org.sonar.db.DbSession; import org.sonar.db.Pagination; import org.sonar.db.organization.OrganizationDto; -import org.sonar.db.rule.RuleDefinitionDto; -import org.sonar.db.rule.RuleParamDto; import org.sonar.server.qualityprofile.index.ActiveRuleIndexer; -import static java.lang.String.format; -import static java.util.Objects.requireNonNull; import static org.sonar.db.Pagination.forPage; /** @@ -63,46 +44,40 @@ public class RegisterQualityProfiles { private final DefinedQProfileRepository definedQProfileRepository; private final DbClient dbClient; + private final DefinedQProfileCreation definedQProfileCreation; private final ActiveRuleIndexer activeRuleIndexer; - private final DefinedQProfileInsert definedQProfileInsert; public RegisterQualityProfiles(DefinedQProfileRepository definedQProfileRepository, - DbClient dbClient, DefinedQProfileInsert definedQProfileInsert, ActiveRuleIndexer activeRuleIndexer) { + DbClient dbClient, DefinedQProfileCreation definedQProfileCreation, ActiveRuleIndexer activeRuleIndexer) { this.definedQProfileRepository = definedQProfileRepository; this.dbClient = dbClient; + this.definedQProfileCreation = definedQProfileCreation; this.activeRuleIndexer = activeRuleIndexer; - this.definedQProfileInsert = definedQProfileInsert; } public void start() { Profiler profiler = Profiler.create(Loggers.get(getClass())).startInfo("Register quality profiles"); - if (definedQProfileRepository.getQProfilesByLanguage().isEmpty()) { - return; - } try (DbSession session = dbClient.openSession(false)) { List changes = new ArrayList<>(); - definedQProfileRepository.getQProfilesByLanguage() - .forEach((key, value) -> registerPerLanguage(session, value, changes)); + definedQProfileRepository.getQProfilesByLanguage().entrySet() + .forEach(entry -> registerPerLanguage(session, entry.getValue(), changes)); activeRuleIndexer.index(changes); profiler.stopDebug(); } } private void registerPerLanguage(DbSession session, List qualityProfiles, List changes) { - qualityProfiles.stream() - .sorted(new SortByParentName(qualityProfiles)) - .forEach(qp -> registerPerQualityProfile(session, qp, changes)); + qualityProfiles.forEach(qp -> registerPerQualityProfile(session, qp, changes)); session.commit(); } private void registerPerQualityProfile(DbSession session, DefinedQProfile qualityProfile, List changes) { LOGGER.info("Register profile {}", qualityProfile.getQProfileName()); - Profiler profiler = Profiler.create(Loggers.get(getClass())); List organizationDtos; while (!(organizationDtos = getOrganizationsWithoutQP(session, qualityProfile)).isEmpty()) { - organizationDtos.forEach(organization -> registerPerQualityProfileAndOrganization(session, qualityProfile, organization, changes, profiler)); + organizationDtos.forEach(organization -> registerPerQualityProfileAndOrganization(session, qualityProfile, organization, changes)); } } @@ -111,80 +86,11 @@ private List getOrganizationsWithoutQP(DbSession session, Defin qualityProfile.getLoadedTemplateType(), PROCESSED_ORGANIZATIONS_BATCH_SIZE); } - private void registerPerQualityProfileAndOrganization(DbSession session, - DefinedQProfile definedQProfile, OrganizationDto organization, List changes, Profiler profiler) { - profiler.start(); - - definedQProfileInsert.create(session, definedQProfile, organization, changes); + private void registerPerQualityProfileAndOrganization(DbSession session, DefinedQProfile qualityProfile, OrganizationDto organization, List changes) { + LOGGER.debug("Register profile {} for organization {}", qualityProfile.getQProfileName(), organization.getKey()); + definedQProfileCreation.create(session, qualityProfile, organization, changes); session.commit(); - - profiler.stopDebug(format("Register profile %s for organization %s", definedQProfile.getQProfileName(), organization.getKey())); - } - - @VisibleForTesting - static class SortByParentName implements Comparator { - private final Map buildersByName; - @VisibleForTesting - final Map depthByBuilder; - - @VisibleForTesting - SortByParentName(Collection builders) { - buildersByName = builders.stream() - .collect(MoreCollectors.uniqueIndex(DefinedQProfile::getName, Function.identity(), builders.size())); - Map depthByBuilder = new HashMap<>(); - builders.forEach(builder -> depthByBuilder.put(builder.getName(), 0)); - builders.forEach(builder -> increaseDepth(buildersByName, depthByBuilder, builder)); - this.depthByBuilder = ImmutableMap.copyOf(depthByBuilder); - } - - private void increaseDepth(Map buildersByName, Map maps, DefinedQProfile builder) { - Optional.ofNullable(builder.getParentQProfileName()) - .ifPresent(parentQProfileName -> { - DefinedQProfile parent = buildersByName.get(parentQProfileName.getName()); - if (parent.getParentQProfileName() != null) { - increaseDepth(buildersByName, maps, parent); - } - maps.put(builder.getName(), maps.get(parent.getName()) + 1); - }); - } - - @Override - public int compare(DefinedQProfile o1, DefinedQProfile o2) { - return depthByBuilder.getOrDefault(o1.getName(), 0) - depthByBuilder.getOrDefault(o2.getName(), 0); - } } - public static class RuleRepository { - private final Map ruleDefinitions; - private final Map> ruleParams; - - public RuleRepository(DbClient dbClient, DbSession session) { - this.ruleDefinitions = dbClient.ruleDao().selectAllDefinitions(session) - .stream() - .collect(Collectors.toMap(RuleDefinitionDto::getKey, Function.identity())); - Map ruleIdsByKey = ruleDefinitions.values() - .stream() - .collect(MoreCollectors.uniqueIndex(RuleDefinitionDto::getId, RuleDefinitionDto::getKey)); - this.ruleParams = new HashMap<>(ruleIdsByKey.size()); - dbClient.ruleDao().selectRuleParamsByRuleKeys(session, ruleDefinitions.keySet()) - .forEach(ruleParam -> ruleParams.compute( - ruleIdsByKey.get(ruleParam.getRuleId()), - (key, value) -> { - if (value == null) { - return ImmutableSet.of(ruleParam); - } - return ImmutableSet.copyOf(Sets.union(value, Collections.singleton(ruleParam))); - })); - } - - public Optional getDefinition(RuleKey ruleKey) { - return Optional.ofNullable(ruleDefinitions.get(requireNonNull(ruleKey, "RuleKey can't be null"))); - } - - public Set getRuleParams(RuleKey ruleKey) { - Set res = ruleParams.get(requireNonNull(ruleKey, "RuleKey can't be null")); - return res == null ? Collections.emptySet() : res; - } - } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/qualityprofile/CachingRuleActivatorTest.java b/server/sonar-server/src/test/java/org/sonar/server/qualityprofile/CachingRuleActivatorTest.java new file mode 100644 index 000000000000..66664b461260 --- /dev/null +++ b/server/sonar-server/src/test/java/org/sonar/server/qualityprofile/CachingRuleActivatorTest.java @@ -0,0 +1,108 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.qualityprofile; + +import java.util.Arrays; +import java.util.stream.Collectors; +import org.junit.Before; +import org.junit.Test; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.db.qualityprofile.QualityProfileDao; +import org.sonar.db.qualityprofile.QualityProfileDto; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Matchers.anyBoolean; +import static org.mockito.Matchers.anyString; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +public class CachingRuleActivatorTest { + private DbClient dbClient = mock(DbClient.class); + private DbSession dbSession = mock(DbSession.class); + private QualityProfileDao qualityProfileDao = mock(QualityProfileDao.class); + private CachingRuleActivator underTest = new CachingRuleActivator(null, dbClient, null, null, null, null, null); + + @Before + public void wire_mocks() throws Exception { + when(dbClient.openSession(anyBoolean())).thenReturn(dbSession); + when(dbClient.qualityProfileDao()).thenReturn(qualityProfileDao); + } + + @Test + public void getChildren_caches_that_qp_has_no_children() { + mockSelectChildrenForKey("no_children"); + + assertThat(underTest.getChildren(dbSession, "no_children")) + .isEmpty(); + assertThat(underTest.getChildren(dbSession, "no_children")) + .isEmpty(); + assertThat(underTest.getChildren(dbSession, "no_children")) + .isEmpty(); + verify(qualityProfileDao, times(1)).selectChildren(eq(dbSession), anyString()); + } + + @Test + public void getChildren_caches_that_sq_has_one_or_more_children() { + mockSelectChildrenForKey("0", "1"); + mockSelectChildrenForKey("1", "2", "3"); + + assertThat(underTest.getChildren(dbSession, "0")) + .extracting(QualityProfileDto::getKey) + .containsExactly("1"); + assertThat(underTest.getChildren(dbSession, "0")) + .extracting(QualityProfileDto::getKey) + .containsExactly("1"); + assertThat(underTest.getChildren(dbSession, "0")) + .extracting(QualityProfileDto::getKey) + .containsExactly("1"); + assertThat(underTest.getChildren(dbSession, "1")) + .extracting(QualityProfileDto::getKey) + .containsExactly("2", "3"); + assertThat(underTest.getChildren(dbSession, "1")) + .extracting(QualityProfileDto::getKey) + .containsExactly("2", "3"); + assertThat(underTest.getChildren(dbSession, "1")) + .extracting(QualityProfileDto::getKey) + .containsExactly("2", "3"); + verify(qualityProfileDao, times(1)).selectChildren(dbSession, "0"); + verify(qualityProfileDao, times(1)).selectChildren(dbSession, "1"); + verifyNoMoreInteractions(qualityProfileDao); + } + + private void mockSelectChildrenForKey(String key, String... children) { + when(qualityProfileDao.selectChildren(dbSession, key)) + .thenReturn(Arrays.stream(children).map(this::dto).collect(Collectors.toList())) + .thenThrow(new IllegalStateException("selectChildren should be called only once for key " + key)); + } + + private QualityProfileDto dto(String key) { + return new QualityProfileDto() { + @Override + public String toString() { + return getKey(); + } + }.setKey(key); + } +} diff --git a/server/sonar-server/src/test/java/org/sonar/server/qualityprofile/RegisterQualityProfilesTest.java b/server/sonar-server/src/test/java/org/sonar/server/qualityprofile/RegisterQualityProfilesTest.java index 9a0cc03abcb1..28abb7f9052c 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/qualityprofile/RegisterQualityProfilesTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/qualityprofile/RegisterQualityProfilesTest.java @@ -25,9 +25,6 @@ import java.util.Iterator; import java.util.List; import java.util.Objects; -import java.util.stream.IntStream; -import javax.annotation.Nullable; -import org.apache.commons.codec.digest.DigestUtils; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -46,11 +43,8 @@ import org.sonar.server.tester.UserSessionRule; import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Matchers.anyBoolean; -import static org.mockito.Matchers.anyList; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; @@ -70,11 +64,11 @@ public class RegisterQualityProfilesTest { private DbClient dbClient = dbTester.getDbClient(); private DbClient mockedDbClient = mock(DbClient.class); private ActiveRuleIndexer mockedActiveRuleIndexer = mock(ActiveRuleIndexer.class); - private DummyDefinedQProfileInsert definedQProfileInsert = new DummyDefinedQProfileInsert(); + private DummyDefinedQProfileCreation definedQProfileCreation = new DummyDefinedQProfileCreation(); private RegisterQualityProfiles underTest = new RegisterQualityProfiles( definedQProfileRepositoryRule, dbClient, - definedQProfileInsert, + definedQProfileCreation, mockedActiveRuleIndexer); @Test @@ -92,9 +86,9 @@ public void no_action_in_DB_nothing_to_index_when_there_is_no_DefinedQProfile() underTest.start(); - assertThat(definedQProfileInsert.getCallLogs()).isEmpty(); - verify(mockedDbClient, times(0)).openSession(anyBoolean()); - verify(mockedActiveRuleIndexer, times(0)).index(anyList()); + assertThat(definedQProfileCreation.getCallLogs()).isEmpty(); + verify(mockedDbClient).openSession(false); + verify(mockedActiveRuleIndexer).index(Collections.emptyList()); verifyNoMoreInteractions(mockedDbClient, mockedActiveRuleIndexer); } @@ -107,7 +101,7 @@ public void start_creates_qps_for_every_organization_in_DB_when_LoadedTemplate_t underTest.start(); - assertThat(definedQProfileInsert.getCallLogs()) + assertThat(definedQProfileCreation.getCallLogs()) .containsExactly( callLog(definedQProfile, dbTester.getDefaultOrganization()), callLog(definedQProfile, organization1), @@ -126,7 +120,7 @@ public void start_creates_qps_only_for_organizations_in_DB_without_loaded_templa underTest.start(); - assertThat(definedQProfileInsert.getCallLogs()) + assertThat(definedQProfileCreation.getCallLogs()) .containsExactly(callLog(definedQProfile, org2)); } @@ -140,7 +134,7 @@ public void start_creates_different_qps_and_their_loaded_templates_if_several_pr underTest.start(); - assertThat(definedQProfileInsert.getCallLogs()) + assertThat(definedQProfileCreation.getCallLogs()) .containsExactly(callLog(definedQProfile2, dbTester.getDefaultOrganization()), callLog(definedQProfile1, dbTester.getDefaultOrganization())); } @@ -155,11 +149,11 @@ public void start_indexes_ActiveRuleChanges_in_order() { ActiveRuleChange ruleChange2 = newActiveRuleChange("2"); ActiveRuleChange ruleChange3 = newActiveRuleChange("3"); ActiveRuleChange ruleChange4 = newActiveRuleChange("4"); - definedQProfileInsert.addChangesPerCall(ruleChange1, ruleChange3); + definedQProfileCreation.addChangesPerCall(ruleChange1, ruleChange3); // no change for second org - definedQProfileInsert.addChangesPerCall(); - definedQProfileInsert.addChangesPerCall(ruleChange2); - definedQProfileInsert.addChangesPerCall(ruleChange4); + definedQProfileCreation.addChangesPerCall(); + definedQProfileCreation.addChangesPerCall(ruleChange2); + definedQProfileCreation.addChangesPerCall(ruleChange4); ArgumentCaptor> indexedChangesCaptor = ArgumentCaptor.forClass((Class>) (Object) List.class); doNothing().when(mockedActiveRuleIndexer).index(indexedChangesCaptor.capture()); @@ -169,52 +163,11 @@ public void start_indexes_ActiveRuleChanges_in_order() { .containsExactly(ruleChange1, ruleChange3, ruleChange2, ruleChange4); } - @Test - public void test_SortByParentName_comporator() { - DefinedQProfile[] builderArray = {newBuilder("A1", null), newBuilder("A2", "A1"), newBuilder("A3", null), newBuilder("A4", "A3"), - newBuilder("A5", "A4"), newBuilder("A6", null)}; - List builders = new ArrayList<>(Arrays.asList(builderArray)); - - IntStream.range(0, 100) - .forEach(i -> { - Collections.shuffle(builders); - RegisterQualityProfiles.SortByParentName comparator = new RegisterQualityProfiles.SortByParentName(builders); - - assertThat(comparator.depthByBuilder.get("A1")).isEqualTo(0); - assertThat(comparator.depthByBuilder.get("A2")).isEqualTo(1); - assertThat(comparator.depthByBuilder.get("A3")).isEqualTo(0); - assertThat(comparator.depthByBuilder.get("A4")).isEqualTo(1); - assertThat(comparator.depthByBuilder.get("A5")).isEqualTo(2); - assertThat(comparator.depthByBuilder.get("A6")).isEqualTo(0); - - builders.sort(comparator); - - verifyParentBeforeChild(builderArray, builders, 0, 1); - verifyParentBeforeChild(builderArray, builders, 2, 3); - verifyParentBeforeChild(builderArray, builders, 3, 4); - verifyParentBeforeChild(builderArray, builders, 2, 4); - }); - } - - private DefinedQProfile newBuilder(String name, @Nullable String parentName) { - return new DefinedQProfile.Builder() - .setName(name) - .setParentName(parentName) - .build(DigestUtils.getMd5Digest()); - } - - private static void verifyParentBeforeChild(DefinedQProfile[] builderArray, List builders, - int parent, int child) { - assertThat(builders.indexOf(builderArray[parent])) - .describedAs(builderArray[4].getName() + " before " + builderArray[child].getName()) - .isLessThan(builders.indexOf(builderArray[child])); - } - private static ActiveRuleChange newActiveRuleChange(String id) { return ActiveRuleChange.createFor(ActiveRuleChange.Type.ACTIVATED, ActiveRuleKey.of(id, RuleKey.of(id + "1", id + "2"))); } - private class DummyDefinedQProfileInsert implements DefinedQProfileInsert { + private class DummyDefinedQProfileCreation implements DefinedQProfileCreation { private List> changesPerCall; private Iterator> changesPerCallIterator; private final List callLogs = new ArrayList<>();