Skip to content

Commit

Permalink
SONAR-11077 access to task characteristics in CE queue
Browse files Browse the repository at this point in the history
  • Loading branch information
Simon Brandhof authored and SonarTech committed Aug 2, 2018
1 parent f72b60c commit d4255d1
Show file tree
Hide file tree
Showing 17 changed files with 211 additions and 146 deletions.
100 changes: 63 additions & 37 deletions server/sonar-ce-common/src/main/java/org/sonar/ce/queue/CeQueueImpl.java
View file
Expand Up @@ -19,7 +19,8 @@
*/ */
package org.sonar.ce.queue; package org.sonar.ce.queue;


import com.google.common.base.Optional; import com.google.common.collect.Multimap;
import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collection; import java.util.Collection;
import java.util.Collections; import java.util.Collections;
Expand All @@ -40,16 +41,17 @@
import org.sonar.db.DbSession; import org.sonar.db.DbSession;
import org.sonar.db.ce.CeActivityDto; import org.sonar.db.ce.CeActivityDto;
import org.sonar.db.ce.CeQueueDto; import org.sonar.db.ce.CeQueueDto;
import org.sonar.db.ce.CeTaskCharacteristicDto;
import org.sonar.db.ce.DeleteIf; import org.sonar.db.ce.DeleteIf;
import org.sonar.db.component.ComponentDto; import org.sonar.db.component.ComponentDto;
import org.sonar.server.organization.DefaultOrganizationProvider; import org.sonar.server.organization.DefaultOrganizationProvider;
import org.sonar.server.property.InternalProperties; import org.sonar.server.property.InternalProperties;


import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Preconditions.checkState;
import static com.google.common.collect.FluentIterable.from;
import static java.util.Collections.singleton; import static java.util.Collections.singleton;
import static org.sonar.ce.queue.CeQueue.SubmitOption.UNIQUE_QUEUE_PER_COMPONENT; import static org.sonar.ce.queue.CeQueue.SubmitOption.UNIQUE_QUEUE_PER_COMPONENT;
import static org.sonar.core.util.stream.MoreCollectors.toEnumSet; import static org.sonar.core.util.stream.MoreCollectors.toEnumSet;
import static org.sonar.core.util.stream.MoreCollectors.uniqueIndex;
import static org.sonar.db.ce.CeQueueDto.Status.PENDING; import static org.sonar.db.ce.CeQueueDto.Status.PENDING;


@ServerSide @ServerSide
Expand Down Expand Up @@ -87,9 +89,15 @@ private java.util.Optional<CeTask> submit(CeTaskSubmit submission, EnumSet<Submi
&& dbClient.ceQueueDao().countByStatusAndComponentUuid(dbSession, PENDING, submission.getComponentUuid()) > 0) { && dbClient.ceQueueDao().countByStatusAndComponentUuid(dbSession, PENDING, submission.getComponentUuid()) > 0) {
return java.util.Optional.empty(); return java.util.Optional.empty();
} }
CeQueueDto dto = addToQueueInDb(dbSession, submission); CeQueueDto taskDto = addToQueueInDb(dbSession, submission);
CeTask task = loadTask(dbSession, dto);
dbSession.commit(); dbSession.commit();

ComponentDto component = null;
String componentUuid = taskDto.getComponentUuid();
if (componentUuid != null) {
component = dbClient.componentDao().selectByUuid(dbSession, componentUuid).orNull();
}
CeTask task = convertToTask(taskDto, submission.getCharacteristics(), component);
return java.util.Optional.of(task); return java.util.Optional.of(task);
} }
} }
Expand All @@ -100,12 +108,12 @@ public List<CeTask> massSubmit(Collection<CeTaskSubmit> submissions, SubmitOptio
return Collections.emptyList(); return Collections.emptyList();
} }


try (DbSession dbSession = dbClient.openSession(true)) { try (DbSession dbSession = dbClient.openSession(false)) {
List<CeQueueDto> ceQueueDtos = submissions.stream() List<CeQueueDto> taskDto = submissions.stream()
.filter(filterBySubmitOptions(options, submissions, dbSession)) .filter(filterBySubmitOptions(options, submissions, dbSession))
.map(submission -> addToQueueInDb(dbSession, submission)) .map(submission -> addToQueueInDb(dbSession, submission))
.collect(Collectors.toList()); .collect(Collectors.toList());
List<CeTask> tasks = loadTasks(dbSession, ceQueueDtos); List<CeTask> tasks = loadTasks(dbSession, taskDto);
dbSession.commit(); dbSession.commit();
return tasks; return tasks;
} }
Expand Down Expand Up @@ -147,37 +155,54 @@ private static EnumSet<SubmitOption> toSet(SubmitOption[] options) {
} }


private CeQueueDto addToQueueInDb(DbSession dbSession, CeTaskSubmit submission) { private CeQueueDto addToQueueInDb(DbSession dbSession, CeTaskSubmit submission) {
for (Map.Entry<String, String> characteristic : submission.getCharacteristics().entrySet()) {
CeTaskCharacteristicDto characteristicDto = new CeTaskCharacteristicDto();
characteristicDto.setUuid(uuidFactory.create());
characteristicDto.setTaskUuid(submission.getUuid());
characteristicDto.setKey(characteristic.getKey());
characteristicDto.setValue(characteristic.getValue());
dbClient.ceTaskCharacteristicsDao().insert(dbSession, characteristicDto);
}

CeQueueDto dto = new CeQueueDto(); CeQueueDto dto = new CeQueueDto();
dto.setUuid(submission.getUuid()); dto.setUuid(submission.getUuid());
dto.setTaskType(submission.getType()); dto.setTaskType(submission.getType());
dto.setComponentUuid(submission.getComponentUuid()); dto.setComponentUuid(submission.getComponentUuid());
dto.setStatus(PENDING); dto.setStatus(PENDING);
dto.setSubmitterUuid(submission.getSubmitterUuid()); dto.setSubmitterUuid(submission.getSubmitterUuid());
dbClient.ceQueueDao().insert(dbSession, dto); dbClient.ceQueueDao().insert(dbSession, dto);
return dto;
}


CeTask loadTask(DbSession dbSession, CeQueueDto dto) { return dto;
String componentUuid = dto.getComponentUuid();
if (componentUuid == null) {
return convertToTask(dto, null);
}
Optional<ComponentDto> componentDto = dbClient.componentDao().selectByUuid(dbSession, componentUuid);
return convertToTask(dto, componentDto.orNull());
} }


private List<CeTask> loadTasks(DbSession dbSession, List<CeQueueDto> dtos) { private List<CeTask> loadTasks(DbSession dbSession, List<CeQueueDto> dtos) {
// load components, if defined
Set<String> componentUuids = dtos.stream() Set<String> componentUuids = dtos.stream()
.map(CeQueueDto::getComponentUuid) .map(CeQueueDto::getComponentUuid)
.filter(Objects::nonNull) .filter(Objects::nonNull)
.collect(Collectors.toSet()); .collect(Collectors.toSet());
Map<String, ComponentDto> componentDtoByUuid = from(dbClient.componentDao() Map<String, ComponentDto> componentsByUuid = dbClient.componentDao()
.selectByUuids(dbSession, componentUuids)) .selectByUuids(dbSession, componentUuids).stream()
.uniqueIndex(ComponentDto::uuid); .collect(uniqueIndex(ComponentDto::uuid));


return dtos.stream() // load characteristics
.map(dto -> convertToTask(dto, dto.getComponentUuid() == null ? null : componentDtoByUuid.get(dto.getComponentUuid()))) // TODO could be avoided, characteristics are already present in submissions
.collect(MoreCollectors.toList(dtos.size())); Set<String> taskUuids = dtos.stream().map(CeQueueDto::getUuid).collect(MoreCollectors.toSet(dtos.size()));
Multimap<String, CeTaskCharacteristicDto> characteristicsByTaskUuid = dbClient.ceTaskCharacteristicsDao()
.selectByTaskUuids(dbSession, taskUuids).stream()
.collect(MoreCollectors.index(CeTaskCharacteristicDto::getTaskUuid));

List<CeTask> result = new ArrayList<>();
for (CeQueueDto dto : dtos) {
ComponentDto component = null;
if (dto.getComponentUuid() != null) {
component = componentsByUuid.get(dto.getComponentUuid());
}
Map<String, String> characteristics = characteristicsByTaskUuid.get(dto.getUuid()).stream()
.collect(uniqueIndex(CeTaskCharacteristicDto::getKey, CeTaskCharacteristicDto::getValue));
result.add(convertToTask(dto, characteristics, component));
}
return result;
} }


@Override @Override
Expand Down Expand Up @@ -215,7 +240,7 @@ public int cancelAll() {
return cancelAll(false); return cancelAll(false);
} }


protected int cancelAll(boolean includeInProgress) { int cancelAll(boolean includeInProgress) {
int count = 0; int count = 0;
try (DbSession dbSession = dbClient.openSession(false)) { try (DbSession dbSession = dbClient.openSession(false)) {
for (CeQueueDto queueDto : dbClient.ceQueueDao().selectAllInAscOrder(dbSession)) { for (CeQueueDto queueDto : dbClient.ceQueueDao().selectAllInAscOrder(dbSession)) {
Expand Down Expand Up @@ -259,24 +284,25 @@ public WorkersPauseStatus getWorkersPauseStatus() {
} }
} }


private CeTask convertToTask(CeQueueDto taskDto, @Nullable ComponentDto componentDto) { CeTask convertToTask(CeQueueDto taskDto, Map<String, String> characteristics, @Nullable ComponentDto component) {
CeTask.Builder builder = new CeTask.Builder(); CeTask.Builder builder = new CeTask.Builder()
builder.setUuid(taskDto.getUuid()); .setUuid(taskDto.getUuid())
builder.setType(taskDto.getTaskType()); .setType(taskDto.getTaskType())
builder.setSubmitterUuid(taskDto.getSubmitterUuid()); .setSubmitterUuid(taskDto.getSubmitterUuid())
String componentUuid = taskDto.getComponentUuid(); .setComponentUuid(taskDto.getComponentUuid())
if (componentUuid != null) { .setCharacteristics(characteristics);
builder.setComponentUuid(componentUuid);
if (componentDto != null) { if (component != null) {
builder.setOrganizationUuid(componentDto.getOrganizationUuid()); builder.setOrganizationUuid(component.getOrganizationUuid());
builder.setComponentKey(componentDto.getDbKey()); builder.setComponentKey(component.getDbKey());
builder.setComponentName(componentDto.name()); builder.setComponentName(component.name());
}
} }

// FIXME this should be set from the CeQueueDto // FIXME this should be set from the CeQueueDto
if (!builder.hasOrganizationUuid()) { if (!builder.hasOrganizationUuid()) {
builder.setOrganizationUuid(defaultOrganizationProvider.get().getUuid()); builder.setOrganizationUuid(defaultOrganizationProvider.get().getUuid());
} }

return builder.build(); return builder.build();
} }


Expand Down
20 changes: 17 additions & 3 deletions server/sonar-ce-common/src/main/java/org/sonar/ce/queue/CeTaskSubmit.java
View file
Expand Up @@ -19,12 +19,14 @@
*/ */
package org.sonar.ce.queue; package org.sonar.ce.queue;


import java.util.Objects; import java.util.Map;
import javax.annotation.CheckForNull; import javax.annotation.CheckForNull;
import javax.annotation.Nullable; import javax.annotation.Nullable;
import javax.annotation.concurrent.Immutable; import javax.annotation.concurrent.Immutable;


import static com.google.common.base.Strings.emptyToNull; import static com.google.common.base.Strings.emptyToNull;
import static java.util.Collections.unmodifiableMap;
import static java.util.Objects.requireNonNull;


@Immutable @Immutable
public final class CeTaskSubmit { public final class CeTaskSubmit {
Expand All @@ -33,12 +35,14 @@ public final class CeTaskSubmit {
private final String type; private final String type;
private final String componentUuid; private final String componentUuid;
private final String submitterUuid; private final String submitterUuid;
private final Map<String, String> characteristics;


private CeTaskSubmit(Builder builder) { private CeTaskSubmit(Builder builder) {
this.uuid = Objects.requireNonNull(emptyToNull(builder.uuid)); this.uuid = requireNonNull(emptyToNull(builder.uuid));
this.type = Objects.requireNonNull(emptyToNull(builder.type)); this.type = requireNonNull(emptyToNull(builder.type));
this.componentUuid = emptyToNull(builder.componentUuid); this.componentUuid = emptyToNull(builder.componentUuid);
this.submitterUuid = emptyToNull(builder.submitterUuid); this.submitterUuid = emptyToNull(builder.submitterUuid);
this.characteristics = unmodifiableMap(requireNonNull(builder.characteristics));
} }


public String getType() { public String getType() {
Expand All @@ -59,11 +63,16 @@ public String getSubmitterUuid() {
return submitterUuid; return submitterUuid;
} }


public Map<String, String> getCharacteristics() {
return characteristics;
}

public static final class Builder { public static final class Builder {
private final String uuid; private final String uuid;
private String type; private String type;
private String componentUuid; private String componentUuid;
private String submitterUuid; private String submitterUuid;
private Map<String, String> characteristics = null;


public Builder(String uuid) { public Builder(String uuid) {
this.uuid = uuid; this.uuid = uuid;
Expand All @@ -88,6 +97,11 @@ public Builder setSubmitterUuid(@Nullable String s) {
return this; return this;
} }


public Builder setCharacteristics(Map<String, String> m) {
this.characteristics = m;
return this;
}

public CeTaskSubmit build() { public CeTaskSubmit build() {
return new CeTaskSubmit(this); return new CeTaskSubmit(this);
} }
Expand Down
12 changes: 7 additions & 5 deletions server/sonar-ce-common/src/test/java/org/sonar/ce/queue/CeQueueImplTest.java
View file
Expand Up @@ -46,6 +46,7 @@


import static com.google.common.collect.ImmutableList.of; import static com.google.common.collect.ImmutableList.of;
import static java.util.Arrays.asList; import static java.util.Arrays.asList;
import static java.util.Collections.emptyMap;
import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic; import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;
import static org.hamcrest.Matchers.startsWith; import static org.hamcrest.Matchers.startsWith;
Expand Down Expand Up @@ -474,11 +475,12 @@ private CeTaskSubmit createTaskSubmit(String type) {
} }


private CeTaskSubmit createTaskSubmit(String type, @Nullable String componentUuid, @Nullable String submitterUuid) { private CeTaskSubmit createTaskSubmit(String type, @Nullable String componentUuid, @Nullable String submitterUuid) {
CeTaskSubmit.Builder submission = underTest.prepareSubmit(); return underTest.prepareSubmit()
submission.setType(type); .setType(type)
submission.setComponentUuid(componentUuid); .setComponentUuid(componentUuid)
submission.setSubmitterUuid(submitterUuid); .setSubmitterUuid(submitterUuid)
return submission.build(); .setCharacteristics(emptyMap())
.build();
} }


private ComponentDto insertComponent(ComponentDto componentDto) { private ComponentDto insertComponent(ComponentDto componentDto) {
Expand Down
6 changes: 5 additions & 1 deletion ...tanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/ExtractReportStepTest.java
View file
Expand Up @@ -57,7 +57,11 @@ public class ExtractReportStepTest {
public DbTester dbTester = DbTester.create(System2.INSTANCE); public DbTester dbTester = DbTester.create(System2.INSTANCE);


private MutableBatchReportDirectoryHolder reportDirectoryHolder = new BatchReportDirectoryHolderImpl(); private MutableBatchReportDirectoryHolder reportDirectoryHolder = new BatchReportDirectoryHolderImpl();
private CeTask ceTask = new CeTask.Builder().setOrganizationUuid("org1").setType(CeTaskTypes.REPORT).setUuid(TASK_UUID).build(); private CeTask ceTask = new CeTask.Builder()
.setOrganizationUuid("org1")
.setType(CeTaskTypes.REPORT)
.setUuid(TASK_UUID)
.build();


private ExtractReportStep underTest = new ExtractReportStep(dbTester.getDbClient(), ceTask, tempFolder, reportDirectoryHolder); private ExtractReportStep underTest = new ExtractReportStep(dbTester.getDbClient(), ceTask, tempFolder, reportDirectoryHolder);


Expand Down
22 changes: 21 additions & 1 deletion server/sonar-ce-task/src/main/java/org/sonar/ce/task/CeTask.java
View file
Expand Up @@ -20,11 +20,15 @@
package org.sonar.ce.task; package org.sonar.ce.task;


import com.google.common.base.MoreObjects; import com.google.common.base.MoreObjects;
import java.util.HashMap;
import java.util.Map;
import javax.annotation.CheckForNull; import javax.annotation.CheckForNull;
import javax.annotation.Nullable; import javax.annotation.Nullable;
import javax.annotation.concurrent.Immutable; import javax.annotation.concurrent.Immutable;


import static com.google.common.base.Strings.emptyToNull; import static com.google.common.base.Strings.emptyToNull;
import static java.util.Collections.emptyMap;
import static java.util.Collections.unmodifiableMap;
import static java.util.Objects.requireNonNull; import static java.util.Objects.requireNonNull;


@Immutable @Immutable
Expand All @@ -37,6 +41,7 @@ public class CeTask {
private final String componentKey; private final String componentKey;
private final String componentName; private final String componentName;
private final String submitterUuid; private final String submitterUuid;
private final Map<String, String> characteristics;


private CeTask(Builder builder) { private CeTask(Builder builder) {
this.organizationUuid = requireNonNull(emptyToNull(builder.organizationUuid), "organizationUuid can't be null nor empty"); this.organizationUuid = requireNonNull(emptyToNull(builder.organizationUuid), "organizationUuid can't be null nor empty");
Expand All @@ -46,6 +51,11 @@ private CeTask(Builder builder) {
this.componentKey = emptyToNull(builder.componentKey); this.componentKey = emptyToNull(builder.componentKey);
this.componentName = emptyToNull(builder.componentName); this.componentName = emptyToNull(builder.componentName);
this.submitterUuid = emptyToNull(builder.submitterUuid); this.submitterUuid = emptyToNull(builder.submitterUuid);
if (builder.characteristics == null) {
this.characteristics = emptyMap();
} else {
this.characteristics = unmodifiableMap(new HashMap<>(builder.characteristics));
}
} }


public String getOrganizationUuid() { public String getOrganizationUuid() {
Expand Down Expand Up @@ -80,6 +90,10 @@ public String getSubmitterUuid() {
return submitterUuid; return submitterUuid;
} }


public Map<String, String> getCharacteristics() {
return characteristics;
}

@Override @Override
public String toString() { public String toString() {
return MoreObjects.toStringHelper(this) return MoreObjects.toStringHelper(this)
Expand Down Expand Up @@ -118,6 +132,7 @@ public static final class Builder {
private String componentKey; private String componentKey;
private String componentName; private String componentName;
private String submitterUuid; private String submitterUuid;
private Map<String, String> characteristics;


public Builder setOrganizationUuid(String organizationUuid) { public Builder setOrganizationUuid(String organizationUuid) {
this.organizationUuid = organizationUuid; this.organizationUuid = organizationUuid;
Expand All @@ -139,7 +154,7 @@ public Builder setType(String type) {
return this; return this;
} }


public Builder setComponentUuid(String componentUuid) { public Builder setComponentUuid(@Nullable String componentUuid) {
this.componentUuid = componentUuid; this.componentUuid = componentUuid;
return this; return this;
} }
Expand All @@ -159,6 +174,11 @@ public Builder setSubmitterUuid(@Nullable String s) {
return this; return this;
} }


public Builder setCharacteristics(@Nullable Map<String, String> m) {
this.characteristics = m;
return this;
}

public CeTask build() { public CeTask build() {
return new CeTask(this); return new CeTask(this);
} }
Expand Down

0 comments on commit d4255d1

Please sign in to comment.