diff --git a/server/sonar-ce-common/src/main/java/org/sonar/ce/queue/CeQueueImpl.java b/server/sonar-ce-common/src/main/java/org/sonar/ce/queue/CeQueueImpl.java index e40c173259d4..c11b41c29208 100644 --- a/server/sonar-ce-common/src/main/java/org/sonar/ce/queue/CeQueueImpl.java +++ b/server/sonar-ce-common/src/main/java/org/sonar/ce/queue/CeQueueImpl.java @@ -19,7 +19,8 @@ */ package org.sonar.ce.queue; -import com.google.common.base.Optional; +import com.google.common.collect.Multimap; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -40,16 +41,17 @@ import org.sonar.db.DbSession; import org.sonar.db.ce.CeActivityDto; import org.sonar.db.ce.CeQueueDto; +import org.sonar.db.ce.CeTaskCharacteristicDto; import org.sonar.db.ce.DeleteIf; import org.sonar.db.component.ComponentDto; import org.sonar.server.organization.DefaultOrganizationProvider; import org.sonar.server.property.InternalProperties; import static com.google.common.base.Preconditions.checkState; -import static com.google.common.collect.FluentIterable.from; import static java.util.Collections.singleton; import static org.sonar.ce.queue.CeQueue.SubmitOption.UNIQUE_QUEUE_PER_COMPONENT; import static org.sonar.core.util.stream.MoreCollectors.toEnumSet; +import static org.sonar.core.util.stream.MoreCollectors.uniqueIndex; import static org.sonar.db.ce.CeQueueDto.Status.PENDING; @ServerSide @@ -87,9 +89,15 @@ private java.util.Optional submit(CeTaskSubmit submission, EnumSet 0) { return java.util.Optional.empty(); } - CeQueueDto dto = addToQueueInDb(dbSession, submission); - CeTask task = loadTask(dbSession, dto); + CeQueueDto taskDto = addToQueueInDb(dbSession, submission); dbSession.commit(); + + ComponentDto component = null; + String componentUuid = taskDto.getComponentUuid(); + if (componentUuid != null) { + component = dbClient.componentDao().selectByUuid(dbSession, componentUuid).orNull(); + } + CeTask task = convertToTask(taskDto, submission.getCharacteristics(), component); return java.util.Optional.of(task); } } @@ -100,12 +108,12 @@ public List massSubmit(Collection submissions, SubmitOptio return Collections.emptyList(); } - try (DbSession dbSession = dbClient.openSession(true)) { - List ceQueueDtos = submissions.stream() + try (DbSession dbSession = dbClient.openSession(false)) { + List taskDto = submissions.stream() .filter(filterBySubmitOptions(options, submissions, dbSession)) .map(submission -> addToQueueInDb(dbSession, submission)) .collect(Collectors.toList()); - List tasks = loadTasks(dbSession, ceQueueDtos); + List tasks = loadTasks(dbSession, taskDto); dbSession.commit(); return tasks; } @@ -147,6 +155,15 @@ private static EnumSet toSet(SubmitOption[] options) { } private CeQueueDto addToQueueInDb(DbSession dbSession, CeTaskSubmit submission) { + for (Map.Entry characteristic : submission.getCharacteristics().entrySet()) { + CeTaskCharacteristicDto characteristicDto = new CeTaskCharacteristicDto(); + characteristicDto.setUuid(uuidFactory.create()); + characteristicDto.setTaskUuid(submission.getUuid()); + characteristicDto.setKey(characteristic.getKey()); + characteristicDto.setValue(characteristic.getValue()); + dbClient.ceTaskCharacteristicsDao().insert(dbSession, characteristicDto); + } + CeQueueDto dto = new CeQueueDto(); dto.setUuid(submission.getUuid()); dto.setTaskType(submission.getType()); @@ -154,30 +171,38 @@ private CeQueueDto addToQueueInDb(DbSession dbSession, CeTaskSubmit submission) dto.setStatus(PENDING); dto.setSubmitterUuid(submission.getSubmitterUuid()); dbClient.ceQueueDao().insert(dbSession, dto); - return dto; - } - CeTask loadTask(DbSession dbSession, CeQueueDto dto) { - String componentUuid = dto.getComponentUuid(); - if (componentUuid == null) { - return convertToTask(dto, null); - } - Optional componentDto = dbClient.componentDao().selectByUuid(dbSession, componentUuid); - return convertToTask(dto, componentDto.orNull()); + return dto; } private List loadTasks(DbSession dbSession, List dtos) { + // load components, if defined Set componentUuids = dtos.stream() .map(CeQueueDto::getComponentUuid) .filter(Objects::nonNull) .collect(Collectors.toSet()); - Map componentDtoByUuid = from(dbClient.componentDao() - .selectByUuids(dbSession, componentUuids)) - .uniqueIndex(ComponentDto::uuid); - - return dtos.stream() - .map(dto -> convertToTask(dto, dto.getComponentUuid() == null ? null : componentDtoByUuid.get(dto.getComponentUuid()))) - .collect(MoreCollectors.toList(dtos.size())); + Map componentsByUuid = dbClient.componentDao() + .selectByUuids(dbSession, componentUuids).stream() + .collect(uniqueIndex(ComponentDto::uuid)); + + // load characteristics + // TODO could be avoided, characteristics are already present in submissions + Set taskUuids = dtos.stream().map(CeQueueDto::getUuid).collect(MoreCollectors.toSet(dtos.size())); + Multimap characteristicsByTaskUuid = dbClient.ceTaskCharacteristicsDao() + .selectByTaskUuids(dbSession, taskUuids).stream() + .collect(MoreCollectors.index(CeTaskCharacteristicDto::getTaskUuid)); + + List result = new ArrayList<>(); + for (CeQueueDto dto : dtos) { + ComponentDto component = null; + if (dto.getComponentUuid() != null) { + component = componentsByUuid.get(dto.getComponentUuid()); + } + Map characteristics = characteristicsByTaskUuid.get(dto.getUuid()).stream() + .collect(uniqueIndex(CeTaskCharacteristicDto::getKey, CeTaskCharacteristicDto::getValue)); + result.add(convertToTask(dto, characteristics, component)); + } + return result; } @Override @@ -215,7 +240,7 @@ public int cancelAll() { return cancelAll(false); } - protected int cancelAll(boolean includeInProgress) { + int cancelAll(boolean includeInProgress) { int count = 0; try (DbSession dbSession = dbClient.openSession(false)) { for (CeQueueDto queueDto : dbClient.ceQueueDao().selectAllInAscOrder(dbSession)) { @@ -259,24 +284,25 @@ public WorkersPauseStatus getWorkersPauseStatus() { } } - private CeTask convertToTask(CeQueueDto taskDto, @Nullable ComponentDto componentDto) { - CeTask.Builder builder = new CeTask.Builder(); - builder.setUuid(taskDto.getUuid()); - builder.setType(taskDto.getTaskType()); - builder.setSubmitterUuid(taskDto.getSubmitterUuid()); - String componentUuid = taskDto.getComponentUuid(); - if (componentUuid != null) { - builder.setComponentUuid(componentUuid); - if (componentDto != null) { - builder.setOrganizationUuid(componentDto.getOrganizationUuid()); - builder.setComponentKey(componentDto.getDbKey()); - builder.setComponentName(componentDto.name()); - } + CeTask convertToTask(CeQueueDto taskDto, Map characteristics, @Nullable ComponentDto component) { + CeTask.Builder builder = new CeTask.Builder() + .setUuid(taskDto.getUuid()) + .setType(taskDto.getTaskType()) + .setSubmitterUuid(taskDto.getSubmitterUuid()) + .setComponentUuid(taskDto.getComponentUuid()) + .setCharacteristics(characteristics); + + if (component != null) { + builder.setOrganizationUuid(component.getOrganizationUuid()); + builder.setComponentKey(component.getDbKey()); + builder.setComponentName(component.name()); } + // FIXME this should be set from the CeQueueDto if (!builder.hasOrganizationUuid()) { builder.setOrganizationUuid(defaultOrganizationProvider.get().getUuid()); } + return builder.build(); } diff --git a/server/sonar-ce-common/src/main/java/org/sonar/ce/queue/CeTaskSubmit.java b/server/sonar-ce-common/src/main/java/org/sonar/ce/queue/CeTaskSubmit.java index c3eb9083d2d6..7ac4669e990b 100644 --- a/server/sonar-ce-common/src/main/java/org/sonar/ce/queue/CeTaskSubmit.java +++ b/server/sonar-ce-common/src/main/java/org/sonar/ce/queue/CeTaskSubmit.java @@ -19,12 +19,14 @@ */ package org.sonar.ce.queue; -import java.util.Objects; +import java.util.Map; import javax.annotation.CheckForNull; import javax.annotation.Nullable; import javax.annotation.concurrent.Immutable; import static com.google.common.base.Strings.emptyToNull; +import static java.util.Collections.unmodifiableMap; +import static java.util.Objects.requireNonNull; @Immutable public final class CeTaskSubmit { @@ -33,12 +35,14 @@ public final class CeTaskSubmit { private final String type; private final String componentUuid; private final String submitterUuid; + private final Map characteristics; private CeTaskSubmit(Builder builder) { - this.uuid = Objects.requireNonNull(emptyToNull(builder.uuid)); - this.type = Objects.requireNonNull(emptyToNull(builder.type)); + this.uuid = requireNonNull(emptyToNull(builder.uuid)); + this.type = requireNonNull(emptyToNull(builder.type)); this.componentUuid = emptyToNull(builder.componentUuid); this.submitterUuid = emptyToNull(builder.submitterUuid); + this.characteristics = unmodifiableMap(requireNonNull(builder.characteristics)); } public String getType() { @@ -59,11 +63,16 @@ public String getSubmitterUuid() { return submitterUuid; } + public Map getCharacteristics() { + return characteristics; + } + public static final class Builder { private final String uuid; private String type; private String componentUuid; private String submitterUuid; + private Map characteristics = null; public Builder(String uuid) { this.uuid = uuid; @@ -88,6 +97,11 @@ public Builder setSubmitterUuid(@Nullable String s) { return this; } + public Builder setCharacteristics(Map m) { + this.characteristics = m; + return this; + } + public CeTaskSubmit build() { return new CeTaskSubmit(this); } diff --git a/server/sonar-ce-common/src/test/java/org/sonar/ce/queue/CeQueueImplTest.java b/server/sonar-ce-common/src/test/java/org/sonar/ce/queue/CeQueueImplTest.java index 8d0c5db35b2b..9d185fcaf3dc 100644 --- a/server/sonar-ce-common/src/test/java/org/sonar/ce/queue/CeQueueImplTest.java +++ b/server/sonar-ce-common/src/test/java/org/sonar/ce/queue/CeQueueImplTest.java @@ -46,6 +46,7 @@ import static com.google.common.collect.ImmutableList.of; import static java.util.Arrays.asList; +import static java.util.Collections.emptyMap; import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic; import static org.assertj.core.api.Assertions.assertThat; import static org.hamcrest.Matchers.startsWith; @@ -474,11 +475,12 @@ private CeTaskSubmit createTaskSubmit(String type) { } private CeTaskSubmit createTaskSubmit(String type, @Nullable String componentUuid, @Nullable String submitterUuid) { - CeTaskSubmit.Builder submission = underTest.prepareSubmit(); - submission.setType(type); - submission.setComponentUuid(componentUuid); - submission.setSubmitterUuid(submitterUuid); - return submission.build(); + return underTest.prepareSubmit() + .setType(type) + .setComponentUuid(componentUuid) + .setSubmitterUuid(submitterUuid) + .setCharacteristics(emptyMap()) + .build(); } private ComponentDto insertComponent(ComponentDto componentDto) { diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/ExtractReportStepTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/ExtractReportStepTest.java index 27def2e2682b..7dd21097f5d6 100644 --- a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/ExtractReportStepTest.java +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/ExtractReportStepTest.java @@ -57,7 +57,11 @@ public class ExtractReportStepTest { public DbTester dbTester = DbTester.create(System2.INSTANCE); private MutableBatchReportDirectoryHolder reportDirectoryHolder = new BatchReportDirectoryHolderImpl(); - private CeTask ceTask = new CeTask.Builder().setOrganizationUuid("org1").setType(CeTaskTypes.REPORT).setUuid(TASK_UUID).build(); + private CeTask ceTask = new CeTask.Builder() + .setOrganizationUuid("org1") + .setType(CeTaskTypes.REPORT) + .setUuid(TASK_UUID) + .build(); private ExtractReportStep underTest = new ExtractReportStep(dbTester.getDbClient(), ceTask, tempFolder, reportDirectoryHolder); diff --git a/server/sonar-ce-task/src/main/java/org/sonar/ce/task/CeTask.java b/server/sonar-ce-task/src/main/java/org/sonar/ce/task/CeTask.java index 68cb82edb9a6..d71f9d036d0e 100644 --- a/server/sonar-ce-task/src/main/java/org/sonar/ce/task/CeTask.java +++ b/server/sonar-ce-task/src/main/java/org/sonar/ce/task/CeTask.java @@ -20,11 +20,15 @@ package org.sonar.ce.task; import com.google.common.base.MoreObjects; +import java.util.HashMap; +import java.util.Map; import javax.annotation.CheckForNull; import javax.annotation.Nullable; import javax.annotation.concurrent.Immutable; import static com.google.common.base.Strings.emptyToNull; +import static java.util.Collections.emptyMap; +import static java.util.Collections.unmodifiableMap; import static java.util.Objects.requireNonNull; @Immutable @@ -37,6 +41,7 @@ public class CeTask { private final String componentKey; private final String componentName; private final String submitterUuid; + private final Map characteristics; private CeTask(Builder builder) { this.organizationUuid = requireNonNull(emptyToNull(builder.organizationUuid), "organizationUuid can't be null nor empty"); @@ -46,6 +51,11 @@ private CeTask(Builder builder) { this.componentKey = emptyToNull(builder.componentKey); this.componentName = emptyToNull(builder.componentName); this.submitterUuid = emptyToNull(builder.submitterUuid); + if (builder.characteristics == null) { + this.characteristics = emptyMap(); + } else { + this.characteristics = unmodifiableMap(new HashMap<>(builder.characteristics)); + } } public String getOrganizationUuid() { @@ -80,6 +90,10 @@ public String getSubmitterUuid() { return submitterUuid; } + public Map getCharacteristics() { + return characteristics; + } + @Override public String toString() { return MoreObjects.toStringHelper(this) @@ -118,6 +132,7 @@ public static final class Builder { private String componentKey; private String componentName; private String submitterUuid; + private Map characteristics; public Builder setOrganizationUuid(String organizationUuid) { this.organizationUuid = organizationUuid; @@ -139,7 +154,7 @@ public Builder setType(String type) { return this; } - public Builder setComponentUuid(String componentUuid) { + public Builder setComponentUuid(@Nullable String componentUuid) { this.componentUuid = componentUuid; return this; } @@ -159,6 +174,11 @@ public Builder setSubmitterUuid(@Nullable String s) { return this; } + public Builder setCharacteristics(@Nullable Map m) { + this.characteristics = m; + return this; + } + public CeTask build() { return new CeTask(this); } diff --git a/server/sonar-ce-task/src/test/java/org/sonar/ce/task/CeTaskTest.java b/server/sonar-ce-task/src/test/java/org/sonar/ce/task/CeTaskTest.java index 842c17de1a32..dbb26c47ca34 100644 --- a/server/sonar-ce-task/src/test/java/org/sonar/ce/task/CeTaskTest.java +++ b/server/sonar-ce-task/src/test/java/org/sonar/ce/task/CeTaskTest.java @@ -19,11 +19,13 @@ */ package org.sonar.ce.task; +import com.google.common.collect.ImmutableMap; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.entry; public class CeTaskTest { @Rule @@ -98,6 +100,7 @@ public void verify_getters() { underTest.setComponentKey("COMPONENT_KEY_1"); underTest.setComponentUuid("COMPONENT_UUID_1"); underTest.setComponentName("The component"); + underTest.setCharacteristics(ImmutableMap.of("k1", "v1", "k2", "v2")); CeTask task = underTest.build(); @@ -108,6 +111,7 @@ public void verify_getters() { assertThat(task.getComponentKey()).isEqualTo("COMPONENT_KEY_1"); assertThat(task.getComponentUuid()).isEqualTo("COMPONENT_UUID_1"); assertThat(task.getComponentName()).isEqualTo("The component"); + assertThat(task.getCharacteristics()).containsExactly(entry("k1", "v1"), entry("k2", "v2")); } @Test @@ -145,4 +149,12 @@ public void equals_and_hashCode_on_uuid() { assertThat(task1.hashCode()).isEqualTo(task1.hashCode()); assertThat(task1.hashCode()).isEqualTo(task1bis.hashCode()); } + + @Test + public void setCharacteristics_null_is_considered_as_empty() { + CeTask task = underTest.setOrganizationUuid("org1").setType("TYPE_1").setUuid("UUID_1") + .setCharacteristics(null) + .build(); + assertThat(task.getCharacteristics()).isEmpty(); + } } diff --git a/server/sonar-ce/src/main/java/org/sonar/ce/queue/InternalCeQueueImpl.java b/server/sonar-ce/src/main/java/org/sonar/ce/queue/InternalCeQueueImpl.java index 743d71336e8e..693b1bf21fd8 100644 --- a/server/sonar-ce/src/main/java/org/sonar/ce/queue/InternalCeQueueImpl.java +++ b/server/sonar-ce/src/main/java/org/sonar/ce/queue/InternalCeQueueImpl.java @@ -24,6 +24,7 @@ import java.io.OutputStream; import java.io.PrintWriter; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import javax.annotation.CheckForNull; @@ -44,11 +45,15 @@ import org.sonar.db.ce.CeActivityDto; import org.sonar.db.ce.CeQueueDao; import org.sonar.db.ce.CeQueueDto; +import org.sonar.db.ce.CeTaskCharacteristicDto; +import org.sonar.db.component.ComponentDto; import org.sonar.server.organization.DefaultOrganizationProvider; import static com.google.common.base.Preconditions.checkArgument; import static java.lang.String.format; +import static java.util.Collections.singletonList; import static java.util.Objects.requireNonNull; +import static org.sonar.core.util.stream.MoreCollectors.uniqueIndex; @ComputeEngineSide public class InternalCeQueueImpl extends CeQueueImpl implements InternalCeQueue { @@ -82,13 +87,22 @@ public Optional peek(String workerUuid) { dbSession.commit(); LOG.debug("{} in progress tasks reset for worker uuid {}", i, workerUuid); } - Optional dto = ceQueueDao.peek(dbSession, workerUuid); - CeTask task = null; - if (dto.isPresent()) { - task = loadTask(dbSession, dto.get()); + Optional opt = ceQueueDao.peek(dbSession, workerUuid); + if (opt.isPresent()) { + CeQueueDto taskDto = opt.get(); + ComponentDto component = null; + String componentUuid = taskDto.getComponentUuid(); + if (componentUuid != null) { + component = dbClient.componentDao().selectByUuid(dbSession, componentUuid).orNull(); + } + Map characteristics = dbClient.ceTaskCharacteristicsDao().selectByTaskUuids(dbSession, singletonList(taskDto.getUuid())).stream() + .collect(uniqueIndex(CeTaskCharacteristicDto::getKey, CeTaskCharacteristicDto::getValue)); + + CeTask task = convertToTask(taskDto, characteristics, component); queueStatus.addInProgress(); + return Optional.of(task); } - return Optional.ofNullable(task); + return Optional.empty(); } } diff --git a/server/sonar-ce/src/main/java/org/sonar/ce/taskprocessor/CeWorker.java b/server/sonar-ce/src/main/java/org/sonar/ce/taskprocessor/CeWorker.java index 6bf1301cec38..74b416fd0bba 100644 --- a/server/sonar-ce/src/main/java/org/sonar/ce/taskprocessor/CeWorker.java +++ b/server/sonar-ce/src/main/java/org/sonar/ce/taskprocessor/CeWorker.java @@ -59,7 +59,7 @@ enum Result { interface ExecutionListener { /** * Called when starting executing a {@link CeTask} (which means: after it's been picked for processing, but before - * the execution of the task by the {@link CeTaskProcessor#process(CeTask)}). + * the execution of the task by the {@link org.sonar.ce.task.taskprocessor.CeTaskProcessor#process(CeTask)}). */ void onStart(CeTask ceTask); diff --git a/server/sonar-ce/src/main/java/org/sonar/ce/taskprocessor/CeWorkerImpl.java b/server/sonar-ce/src/main/java/org/sonar/ce/taskprocessor/CeWorkerImpl.java index 861f1b7c058a..9fc7e9ec281c 100644 --- a/server/sonar-ce/src/main/java/org/sonar/ce/taskprocessor/CeWorkerImpl.java +++ b/server/sonar-ce/src/main/java/org/sonar/ce/taskprocessor/CeWorkerImpl.java @@ -179,17 +179,13 @@ private void finalizeTask(CeTask task, Profiler ceProfiler, CeActivityDto.Status } private static Profiler startLogProfiler(CeTask task) { - Profiler profiler = Profiler.create(LOG); - profiler + return Profiler.create(LOG) .logTimeLast(true) .addContext("project", task.getComponentKey()) .addContext("type", task.getType()) - .addContext("id", task.getUuid()); - String submitterLogin = task.getSubmitterUuid(); - if (submitterLogin != null) { - profiler.addContext("submitter", submitterLogin); - } - return profiler.startInfo("Execute task"); + .addContext("id", task.getUuid()) + .addContext("submitter", task.getSubmitterUuid()) + .startInfo("Execute task"); } private static void stopLogProfiler(Profiler profiler, CeActivityDto.Status status) { diff --git a/server/sonar-ce/src/test/java/org/sonar/ce/queue/InternalCeQueueImplTest.java b/server/sonar-ce/src/test/java/org/sonar/ce/queue/InternalCeQueueImplTest.java index 74c38b92df52..b31efe0a60de 100644 --- a/server/sonar-ce/src/test/java/org/sonar/ce/queue/InternalCeQueueImplTest.java +++ b/server/sonar-ce/src/test/java/org/sonar/ce/queue/InternalCeQueueImplTest.java @@ -52,6 +52,7 @@ import org.sonar.server.organization.DefaultOrganizationProvider; import static java.util.Arrays.asList; +import static java.util.Collections.emptyMap; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -588,11 +589,12 @@ private CeTaskSubmit createTaskSubmit(String type) { } private CeTaskSubmit createTaskSubmit(String type, @Nullable String componentUuid, @Nullable String submitterUuid) { - CeTaskSubmit.Builder submission = underTest.prepareSubmit(); - submission.setType(type); - submission.setComponentUuid(componentUuid); - submission.setSubmitterUuid(submitterUuid); - return submission.build(); + return underTest.prepareSubmit() + .setType(type) + .setComponentUuid(componentUuid) + .setSubmitterUuid(submitterUuid) + .setCharacteristics(emptyMap()) + .build(); } private CeTaskResult newTaskResult(@Nullable String analysisUuid) { diff --git a/server/sonar-ce/src/test/java/org/sonar/ce/taskprocessor/CeTaskProcessorRepositoryImplTest.java b/server/sonar-ce/src/test/java/org/sonar/ce/taskprocessor/CeTaskProcessorRepositoryImplTest.java index 4108af811830..acd806917121 100644 --- a/server/sonar-ce/src/test/java/org/sonar/ce/taskprocessor/CeTaskProcessorRepositoryImplTest.java +++ b/server/sonar-ce/src/test/java/org/sonar/ce/taskprocessor/CeTaskProcessorRepositoryImplTest.java @@ -111,7 +111,9 @@ private static CeTask createCeTask(String ceTaskType, String key) { .setOrganizationUuid("org1") .setType(ceTaskType) .setUuid("task_uuid_" + key) - .setComponentKey(key).setComponentUuid("uuid_" + key).setComponentName("name_" + key) + .setComponentKey(key) + .setComponentUuid("uuid_" + key) + .setComponentName("name_" + key) .build(); } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeTaskCharacteristicDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeTaskCharacteristicDao.java index 1a9f3547cd6f..8116c6830569 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeTaskCharacteristicDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeTaskCharacteristicDao.java @@ -32,11 +32,15 @@ public class CeTaskCharacteristicDao implements Dao { public void insert(DbSession dbSession, Collection characteristics) { for (CeTaskCharacteristicDto dto : characteristics) { - mapper(dbSession).insert(dto); + insert(dbSession, dto); } } - public List selectByTaskUuids(DbSession dbSession, List taskUuids) { + public void insert(DbSession dbSession, CeTaskCharacteristicDto dto) { + mapper(dbSession).insert(dto); + } + + public List selectByTaskUuids(DbSession dbSession, Collection taskUuids) { return executeLargeInputs(taskUuids, uuid -> mapper(dbSession).selectByTaskUuids(uuid)); } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/issue/IssueDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/issue/IssueDao.java index 33b2a60c1e89..82e08406cf52 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/issue/IssueDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/issue/IssueDao.java @@ -53,7 +53,7 @@ public IssueDto selectOrFailByKey(DbSession session, String key) { * if input keys contain multiple occurrences of a key. *

Results may be in a different order as input keys.

*/ - public List selectByKeys(final DbSession session, Collection keys) { + public List selectByKeys(DbSession session, Collection keys) { return executeLargeInputs(keys, mapper(session)::selectByKeys); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/ce/queue/ReportSubmitter.java b/server/sonar-server/src/main/java/org/sonar/server/ce/queue/ReportSubmitter.java index 29e95c417798..73ecb04f18ba 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/ce/queue/ReportSubmitter.java +++ b/server/sonar-server/src/main/java/org/sonar/server/ce/queue/ReportSubmitter.java @@ -22,7 +22,6 @@ import com.google.common.base.Optional; import java.io.InputStream; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Map; import javax.annotation.Nullable; @@ -33,11 +32,8 @@ import org.sonar.ce.queue.CeTaskSubmit; import org.sonar.ce.task.CeTask; import org.sonar.core.component.ComponentKeys; -import org.sonar.core.util.UuidFactory; -import org.sonar.core.util.stream.MoreCollectors; import org.sonar.db.DbClient; import org.sonar.db.DbSession; -import org.sonar.db.ce.CeTaskCharacteristicDto; import org.sonar.db.ce.CeTaskTypes; import org.sonar.db.component.ComponentDto; import org.sonar.db.organization.OrganizationDto; @@ -64,22 +60,16 @@ public class ReportSubmitter { private final ComponentUpdater componentUpdater; private final PermissionTemplateService permissionTemplateService; private final DbClient dbClient; - private final UuidFactory uuidFactory; public ReportSubmitter(CeQueue queue, UserSession userSession, ComponentUpdater componentUpdater, - PermissionTemplateService permissionTemplateService, UuidFactory uuidFactory, DbClient dbClient) { + PermissionTemplateService permissionTemplateService, DbClient dbClient) { this.queue = queue; this.userSession = userSession; this.componentUpdater = componentUpdater; this.permissionTemplateService = permissionTemplateService; - this.uuidFactory = uuidFactory; this.dbClient = dbClient; } - public CeTask submit(String organizationKey, String projectKey, @Nullable String projectBranch, @Nullable String projectName, InputStream reportInput) { - return submit(organizationKey, projectKey, projectBranch, projectName, Collections.emptyMap(), reportInput); - } - /** * @throws NotFoundException if the organization with the specified key does not exist * @throws IllegalArgumentException if the organization with the specified key is not the organization of the specified project (when it already exists in DB) @@ -169,29 +159,17 @@ private ComponentDto createProject(DbSession dbSession, OrganizationDto organiza return componentUpdater.create(dbSession, newProject, userId); } - private CeTask submitReport(DbSession dbSession, InputStream reportInput, ComponentDto project, Map characteristicsMap) { + private CeTask submitReport(DbSession dbSession, InputStream reportInput, ComponentDto project, Map characteristics) { CeTaskSubmit.Builder submit = queue.prepareSubmit(); - List characteristics = characteristicsMap.entrySet().stream() - .map(e -> toDto(submit.getUuid(), e.getKey(), e.getValue())) - .collect(MoreCollectors.toList(characteristicsMap.size())); // the report file must be saved before submitting the task dbClient.ceTaskInputDao().insert(dbSession, submit.getUuid(), reportInput); - dbClient.ceTaskCharacteristicsDao().insert(dbSession, characteristics); dbSession.commit(); submit.setType(CeTaskTypes.REPORT); submit.setComponentUuid(project.uuid()); submit.setSubmitterUuid(userSession.getUuid()); + submit.setCharacteristics(characteristics); return queue.submit(submit.build()); } - - private CeTaskCharacteristicDto toDto(String taskUuid, String key, String value) { - CeTaskCharacteristicDto dto = new CeTaskCharacteristicDto(); - dto.setTaskUuid(taskUuid); - dto.setKey(key); - dto.setValue(value); - dto.setUuid(uuidFactory.create()); - return dto; - } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/ce/queue/ReportSubmitterTest.java b/server/sonar-server/src/test/java/org/sonar/server/ce/queue/ReportSubmitterTest.java index 5eec218d72f7..6ad3033a68b3 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/ce/queue/ReportSubmitterTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/ce/queue/ReportSubmitterTest.java @@ -19,8 +19,8 @@ */ package org.sonar.server.ce.queue; -import java.util.HashMap; -import java.util.List; +import com.google.common.collect.ImmutableMap; +import java.nio.charset.StandardCharsets; import java.util.Map; import org.apache.commons.io.IOUtils; import org.junit.Before; @@ -34,8 +34,6 @@ import org.sonar.ce.queue.CeQueueImpl; import org.sonar.ce.queue.CeTaskSubmit; import org.sonar.core.permission.GlobalPermissions; -import org.sonar.core.util.SequenceUuidFactory; -import org.sonar.core.util.UuidFactory; import org.sonar.db.DbSession; import org.sonar.db.DbTester; import org.sonar.db.ce.CeTaskCharacteristicDto; @@ -55,9 +53,10 @@ import org.sonar.server.tester.UserSessionRule; import static java.lang.String.format; -import static java.util.Collections.singletonList; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Collections.emptyMap; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.tuple; +import static org.assertj.core.api.Assertions.entry; import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.argThat; @@ -93,9 +92,8 @@ public class ReportSubmitterTest { private ComponentUpdater componentUpdater = mock(ComponentUpdater.class); private PermissionTemplateService permissionTemplateService = mock(PermissionTemplateService.class); private FavoriteUpdater favoriteUpdater = mock(FavoriteUpdater.class); - private UuidFactory uuidFactory = new SequenceUuidFactory(); - private ReportSubmitter underTest = new ReportSubmitter(queue, userSession, componentUpdater, permissionTemplateService, uuidFactory, db.getDbClient()); + private ReportSubmitter underTest = new ReportSubmitter(queue, userSession, componentUpdater, permissionTemplateService, db.getDbClient()); @Before public void setUp() throws Exception { @@ -104,7 +102,7 @@ public void setUp() throws Exception { } @Test - public void submit_inserts_characteristics() { + public void submit_stores_report() { userSession .addPermission(OrganizationPermission.SCAN, db.getDefaultOrganization().getUuid()) .addPermission(PROVISION_PROJECTS, db.getDefaultOrganization()); @@ -114,22 +112,15 @@ public void submit_inserts_characteristics() { when(componentUpdater.create(any(), any(), any())).thenReturn(project); when(permissionTemplateService.wouldUserHaveScanPermissionWithDefaultTemplate(any(), eq(defaultOrganizationUuid), any(), eq(PROJECT_KEY), eq(Qualifiers.PROJECT))) - .thenReturn(true); + .thenReturn(true); - Map taskCharacteristics = new HashMap<>(); - taskCharacteristics.put("incremental", "true"); - taskCharacteristics.put("pr", "mypr"); - - underTest.submit(defaultOrganizationKey, PROJECT_KEY, null, PROJECT_NAME, taskCharacteristics, IOUtils.toInputStream("{binary}")); + Map taskCharacteristics = ImmutableMap.of(CeTaskCharacteristicDto.PULL_REQUEST, "123"); + underTest.submit(defaultOrganizationKey, PROJECT_KEY, null, PROJECT_NAME, taskCharacteristics, IOUtils.toInputStream("{binary}", UTF_8)); ArgumentCaptor submittedTask = ArgumentCaptor.forClass(CeTaskSubmit.class); verify(queue).submit(submittedTask.capture()); - String taskUuid = submittedTask.getValue().getUuid(); - - List insertedCharacteristics = db.getDbClient().ceTaskCharacteristicsDao().selectByTaskUuids(db.getSession(), singletonList(taskUuid)); - assertThat(insertedCharacteristics) - .extracting(CeTaskCharacteristicDto::getKey, CeTaskCharacteristicDto::getValue) - .containsOnly(tuple("incremental", "true"), tuple("pr", "mypr")); + assertThat(submittedTask.getValue().getCharacteristics()) + .containsExactly(entry("pullRequest", "123")); } @Test @@ -140,16 +131,15 @@ public void submit_a_report_on_existing_project() { mockSuccessfulPrepareSubmitCall(); - underTest.submit(defaultOrganizationKey, project.getDbKey(), null, project.name(), IOUtils.toInputStream("{binary}")); + underTest.submit(defaultOrganizationKey, project.getDbKey(), null, project.name(), emptyMap(), IOUtils.toInputStream("{binary}", StandardCharsets.UTF_8)); verifyReportIsPersisted(TASK_UUID); verifyZeroInteractions(permissionTemplateService); verifyZeroInteractions(favoriteUpdater); - verify(queue).submit(argThat(submit -> - submit.getType().equals(CeTaskTypes.REPORT) - && submit.getComponentUuid().equals(project.uuid()) - && submit.getSubmitterUuid().equals(user.getUuid()) - && submit.getUuid().equals(TASK_UUID))); + verify(queue).submit(argThat(submit -> submit.getType().equals(CeTaskTypes.REPORT) + && submit.getComponentUuid().equals(project.uuid()) + && submit.getSubmitterUuid().equals(user.getUuid()) + && submit.getUuid().equals(TASK_UUID))); } @Test @@ -164,14 +154,13 @@ public void provision_project_if_does_not_exist() { when(componentUpdater.create(any(), any(), isNull())).thenReturn(createdProject); when( permissionTemplateService.wouldUserHaveScanPermissionWithDefaultTemplate(any(), eq(organization.getUuid()), any(), eq(PROJECT_KEY), eq(Qualifiers.PROJECT))) - .thenReturn(true); + .thenReturn(true); when(permissionTemplateService.hasDefaultTemplateWithPermissionOnProjectCreator(any(), eq(organization.getUuid()), any())).thenReturn(true); - underTest.submit(organization.getKey(), PROJECT_KEY, null, PROJECT_NAME, IOUtils.toInputStream("{binary}")); + underTest.submit(organization.getKey(), PROJECT_KEY, null, PROJECT_NAME, emptyMap(), IOUtils.toInputStream("{binary}")); verifyReportIsPersisted(TASK_UUID); - verify(queue).submit(argThat(submit -> - submit.getType().equals(CeTaskTypes.REPORT) && submit.getComponentUuid().equals(PROJECT_UUID) && submit.getUuid().equals(TASK_UUID))); + verify(queue).submit(argThat(submit -> submit.getType().equals(CeTaskTypes.REPORT) && submit.getComponentUuid().equals(PROJECT_UUID) && submit.getUuid().equals(TASK_UUID))); } @Test @@ -185,10 +174,10 @@ public void no_favorite_when_no_project_creator_permission_on_permission_templat when(componentUpdater.create(any(), any(), isNull())).thenReturn(createdProject); when(permissionTemplateService.wouldUserHaveScanPermissionWithDefaultTemplate(any(), eq(defaultOrganizationUuid), any(), eq(PROJECT_KEY), eq(Qualifiers.PROJECT))) - .thenReturn(true); + .thenReturn(true); when(permissionTemplateService.hasDefaultTemplateWithPermissionOnProjectCreator(any(), eq(defaultOrganizationUuid), any())).thenReturn(false); - underTest.submit(defaultOrganizationKey, PROJECT_KEY, null, PROJECT_NAME, IOUtils.toInputStream("{binary}")); + underTest.submit(defaultOrganizationKey, PROJECT_KEY, null, PROJECT_NAME, emptyMap(), IOUtils.toInputStream("{binary}")); verifyZeroInteractions(favoriteUpdater); } @@ -204,9 +193,9 @@ public void submit_a_report_on_new_project_with_scan_permission_on_organization( when(componentUpdater.create(any(), any(), any())).thenReturn(project); when(permissionTemplateService.wouldUserHaveScanPermissionWithDefaultTemplate(any(), eq(defaultOrganizationUuid), any(), eq(PROJECT_KEY), eq(Qualifiers.PROJECT))) - .thenReturn(true); + .thenReturn(true); - underTest.submit(defaultOrganizationKey, PROJECT_KEY, null, PROJECT_NAME, IOUtils.toInputStream("{binary}")); + underTest.submit(defaultOrganizationKey, PROJECT_KEY, null, PROJECT_NAME, emptyMap(), IOUtils.toInputStream("{binary}")); verify(queue).submit(any(CeTaskSubmit.class)); } @@ -219,7 +208,7 @@ public void user_with_scan_permission_on_organization_is_allowed_to_submit_a_rep mockSuccessfulPrepareSubmitCall(); - underTest.submit(org.getKey(), project.getDbKey(), null, project.name(), IOUtils.toInputStream("{binary}")); + underTest.submit(org.getKey(), project.getDbKey(), null, project.name(), emptyMap(), IOUtils.toInputStream("{binary}")); verify(queue).submit(any(CeTaskSubmit.class)); } @@ -231,7 +220,7 @@ public void submit_a_report_on_existing_project_with_project_scan_permission() { mockSuccessfulPrepareSubmitCall(); - underTest.submit(defaultOrganizationKey, project.getDbKey(), null, project.name(), IOUtils.toInputStream("{binary}")); + underTest.submit(defaultOrganizationKey, project.getDbKey(), null, project.name(), emptyMap(), IOUtils.toInputStream("{binary}")); verify(queue).submit(any(CeTaskSubmit.class)); } @@ -249,7 +238,7 @@ public void project_branch_must_not_benefit_from_the_scan_permission_on_main_pro ComponentDto branchProject = db.components().insertPrivateProject(p -> p.setDbKey(mainProject.getDbKey() + ":" + branchName)); expectedException.expect(ForbiddenException.class); - underTest.submit(defaultOrganizationKey, mainProject.getDbKey(), branchName, PROJECT_NAME, IOUtils.toInputStream("{binary}")); + underTest.submit(defaultOrganizationKey, mainProject.getDbKey(), branchName, PROJECT_NAME,emptyMap(), IOUtils.toInputStream("{binary}")); } @Test @@ -257,7 +246,7 @@ public void fail_with_NotFoundException_if_organization_with_specified_key_does_ expectedException.expect(NotFoundException.class); expectedException.expectMessage("Organization with key 'fop' does not exist"); - underTest.submit("fop", PROJECT_KEY, null, null, null /* method will fail before parameter is used */); + underTest.submit("fop", PROJECT_KEY, null, null, emptyMap(), null /* method will fail before parameter is used */); } @Test @@ -267,7 +256,7 @@ public void fail_with_organizationKey_does_not_match_organization_of_specified_c ComponentDto project = db.components().insertPrivateProject(organization); mockSuccessfulPrepareSubmitCall(); - underTest.submit(organization.getKey(), project.getDbKey(), null, project.name(), IOUtils.toInputStream("{binary}")); + underTest.submit(organization.getKey(), project.getDbKey(), null, project.name(), emptyMap(), IOUtils.toInputStream("{binary}")); } @Test @@ -279,7 +268,7 @@ public void fail_if_component_is_not_a_project() { expectedException.expect(BadRequestException.class); expectedException.expectMessage(format("Component '%s' is not a project", component.getKey())); - underTest.submit(defaultOrganizationKey, component.getDbKey(), null, component.name(), IOUtils.toInputStream("{binary}")); + underTest.submit(defaultOrganizationKey, component.getDbKey(), null, component.name(), emptyMap(), IOUtils.toInputStream("{binary}")); } @Test @@ -290,12 +279,12 @@ public void fail_if_project_key_already_exists_as_module() { mockSuccessfulPrepareSubmitCall(); try { - underTest.submit(defaultOrganizationKey, module.getDbKey(), null, module.name(), IOUtils.toInputStream("{binary}")); + underTest.submit(defaultOrganizationKey, module.getDbKey(), null, module.name(), emptyMap(), IOUtils.toInputStream("{binary}")); fail(); } catch (BadRequestException e) { assertThat(e.errors()).contains( format("The project '%s' is already defined in SonarQube but as a module of project '%s'. " + - "If you really want to stop directly analysing project '%s', please first delete it from SonarQube and then relaunch the analysis of project '%s'.", + "If you really want to stop directly analysing project '%s', please first delete it from SonarQube and then relaunch the analysis of project '%s'.", module.getKey(), project.getKey(), project.getKey(), module.getKey())); } } @@ -304,7 +293,7 @@ public void fail_if_project_key_already_exists_as_module() { public void fail_with_forbidden_exception_when_no_scan_permission() { expectedException.expect(ForbiddenException.class); - underTest.submit(defaultOrganizationKey, PROJECT_KEY, null, PROJECT_NAME, IOUtils.toInputStream("{binary}")); + underTest.submit(defaultOrganizationKey, PROJECT_KEY, null, PROJECT_NAME, emptyMap(), IOUtils.toInputStream("{binary}")); } @Test @@ -315,7 +304,7 @@ public void fail_with_forbidden_exception_on_new_project_when_only_project_scan_ when(componentUpdater.create(any(DbSession.class), any(NewComponent.class), eq(null))).thenReturn(new ComponentDto().setUuid(PROJECT_UUID).setDbKey(PROJECT_KEY)); expectedException.expect(ForbiddenException.class); - underTest.submit(defaultOrganizationKey, PROJECT_KEY, null, PROJECT_NAME, IOUtils.toInputStream("{binary}")); + underTest.submit(defaultOrganizationKey, PROJECT_KEY, null, PROJECT_NAME, emptyMap(), IOUtils.toInputStream("{binary}")); } private void verifyReportIsPersisted(String taskUuid) { diff --git a/server/sonar-server/src/test/java/org/sonar/server/ce/ws/CancelActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/ce/ws/CancelActionTest.java index 3765b154bc27..2236a47cc6a6 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/ce/ws/CancelActionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/ce/ws/CancelActionTest.java @@ -40,6 +40,7 @@ import org.sonar.server.tester.UserSessionRule; import org.sonar.server.ws.WsActionTester; +import static java.util.Collections.emptyMap; import static org.assertj.core.api.Assertions.assertThat; public class CancelActionTest { @@ -172,10 +173,11 @@ private void logInAsSystemAdministrator() { } private CeQueueDto createTaskSubmit(@Nullable String componentUuid) { - CeTaskSubmit.Builder submission = queue.prepareSubmit(); - submission.setType(CeTaskTypes.REPORT); - submission.setComponentUuid(componentUuid); - submission.setSubmitterUuid(null); + CeTaskSubmit.Builder submission = queue.prepareSubmit() + .setType(CeTaskTypes.REPORT) + .setComponentUuid(componentUuid) + .setSubmitterUuid(null) + .setCharacteristics(emptyMap()); CeTask task = queue.submit(submission.build()); return db.getDbClient().ceQueueDao().selectByUuid(db.getSession(), task.getUuid()).get(); } diff --git a/server/sonar-server/src/test/java/org/sonar/server/ce/ws/SubmitActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/ce/ws/SubmitActionTest.java index 8892098fe9f8..95b97a3eb44b 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/ce/ws/SubmitActionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/ce/ws/SubmitActionTest.java @@ -22,7 +22,6 @@ import com.google.common.base.Strings; import java.io.ByteArrayInputStream; import java.io.InputStream; -import java.util.Arrays; import java.util.Map; import org.junit.Before; import org.junit.Test; @@ -40,6 +39,7 @@ import org.sonarqube.ws.Ce; import org.sonarqube.ws.MediaTypes; +import static java.util.Arrays.asList; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.entry; import static org.mockito.ArgumentMatchers.any; @@ -102,7 +102,7 @@ public void submit_task_with_multiple_characteristics() { Ce.SubmitResponse submitResponse = tester.newRequest() .setParam("projectKey", "my_project") .setParam("projectName", "My Project") - .setMultiParam("characteristic", Arrays.asList(characteristics)) + .setMultiParam("characteristic", asList(characteristics)) .setPart("report", new ByteArrayInputStream("{binary}".getBytes()), "foo.bar") .setMethod("POST") .executeProtobuf(Ce.SubmitResponse.class);