Skip to content

Commit

Permalink
Use 'distinct' when displaying objects (MID-3955)
Browse files Browse the repository at this point in the history 
The GUI uses 'distinct' search option when displaying objects, unless
disabled in the system configuration.

Important change in repo: query interpreter now optimizes usage of
'distinct'. If there are no joins, distinct is not applied even if
requested. This heuristic would need to be changed if it would be
found to be ineffective (see e.g.
QueryInterpreter2Test.test0350QuerySubtreeDistinctCount).
  • Loading branch information
@mederly
mederly committed Apr 11, 2018
1 parent 4957773 commit cd3cbcb
Show file tree
Hide file tree
Showing 17 changed files with 252 additions and 90 deletions.
2 changes: 2 additions & 0 deletions gui/admin-gui/src/main/java/com/evolveum/midpoint/gui/api/component/ObjectListPanel.java
View file
Expand Up @@ -57,6 +57,7 @@
import com.evolveum.midpoint.web.component.util.SelectableBean;
import com.evolveum.midpoint.web.session.PageStorage;
import com.evolveum.midpoint.web.session.UserProfileStorage.TableId;
import org.jetbrains.annotations.NotNull;

/**
* @author katkav
Expand Down Expand Up @@ -361,6 +362,7 @@ public SelectableBean<O> createDataObjectWrapper(O obj) {
return bean;
}

@NotNull
@Override
protected List<ObjectOrdering> createObjectOrderings(SortParam<String> sortParam) {
List<ObjectOrdering> customOrdering = createCustomOrdering(sortParam);
Expand Down
12 changes: 12 additions & 0 deletions gui/admin-gui/src/main/java/com/evolveum/midpoint/gui/api/util/WebComponentUtil.java
View file
Expand Up @@ -416,6 +416,18 @@ public static void safeResultCleanup(OperationResult result, Trace logger) {
}
}

public static GuiObjectListType getDefaultGuiObjectListType(PageBase pageBase) {
AdminGuiConfigurationType config = pageBase.getPrincipal().getAdminGuiConfiguration();
if (config == null) {
return null;
}
GuiObjectListsType lists = config.getObjectLists();
if (lists == null) {
return null;
}
return lists.getDefault();
}

public enum Channel {
// TODO: move this to schema component
LIVE_SYNC(SchemaConstants.CHANGE_CHANNEL_LIVE_SYNC_URI),
Expand Down
59 changes: 33 additions & 26 deletions ...-gui/src/main/java/com/evolveum/midpoint/web/component/data/BaseSortableDataProvider.java
View file
Expand Up @@ -17,6 +17,7 @@
package com.evolveum.midpoint.web.component.data;

import com.evolveum.midpoint.gui.api.page.PageBase;
import com.evolveum.midpoint.gui.api.util.WebComponentUtil;
import com.evolveum.midpoint.model.api.*;
import com.evolveum.midpoint.prism.PrismContext;
import com.evolveum.midpoint.prism.path.ItemPath;
Expand All @@ -25,7 +26,9 @@
import com.evolveum.midpoint.prism.query.ObjectQuery;
import com.evolveum.midpoint.prism.query.OrderDirection;
import com.evolveum.midpoint.repo.api.RepositoryService;
import com.evolveum.midpoint.schema.GetOperationOptions;
import com.evolveum.midpoint.schema.SchemaConstantsGenerated;
import com.evolveum.midpoint.schema.SelectorOptions;
import com.evolveum.midpoint.schema.result.OperationResult;
import com.evolveum.midpoint.task.api.TaskManager;
import com.evolveum.midpoint.util.logging.Trace;
Expand All @@ -34,8 +37,8 @@
import com.evolveum.midpoint.web.security.MidPointApplication;
import com.evolveum.midpoint.wf.api.WorkflowManager;
import com.evolveum.midpoint.xml.ns._public.common.common_3.AdminGuiConfigurationType;
import com.evolveum.midpoint.xml.ns._public.common.common_3.DistinctSearchOptionType;
import com.evolveum.midpoint.xml.ns._public.common.common_3.GuiObjectListType;
import com.evolveum.midpoint.xml.ns._public.common.common_3.GuiObjectListsType;
import org.apache.commons.lang.Validate;
import org.apache.wicket.Component;
import org.apache.wicket.extensions.markup.html.repeater.data.sort.SortOrder;
Expand All @@ -45,6 +48,7 @@
import org.apache.wicket.model.IModel;
import org.apache.wicket.model.Model;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;

import javax.xml.namespace.QName;
import java.io.Serializable;
Expand Down Expand Up @@ -93,47 +97,47 @@ public BaseSortableDataProvider(Component component, boolean useCache, boolean u
}

protected ModelService getModel() {
MidPointApplication application = (MidPointApplication) MidPointApplication.get();
MidPointApplication application = MidPointApplication.get();
return application.getModel();
}

protected RepositoryService getRepositoryService() {
MidPointApplication application = (MidPointApplication) MidPointApplication.get();
MidPointApplication application = MidPointApplication.get();
return application.getRepositoryService();
}

protected TaskManager getTaskManager() {
MidPointApplication application = (MidPointApplication) MidPointApplication.get();
MidPointApplication application = MidPointApplication.get();
return application.getTaskManager();
}

protected PrismContext getPrismContext() {
MidPointApplication application = (MidPointApplication) MidPointApplication.get();
MidPointApplication application = MidPointApplication.get();
return application.getPrismContext();
}

protected TaskService getTaskService() {
MidPointApplication application = (MidPointApplication) MidPointApplication.get();
MidPointApplication application = MidPointApplication.get();
return application.getTaskService();
}

protected ModelInteractionService getModelInteractionService() {
MidPointApplication application = (MidPointApplication) MidPointApplication.get();
MidPointApplication application = MidPointApplication.get();
return application.getModelInteractionService();
}

protected WorkflowService getWorkflowService() {
MidPointApplication application = (MidPointApplication) MidPointApplication.get();
MidPointApplication application = MidPointApplication.get();
return application.getWorkflowService();
}

protected ModelAuditService getAuditService() {
MidPointApplication application = (MidPointApplication) MidPointApplication.get();
MidPointApplication application = MidPointApplication.get();
return application.getAuditService();
}

protected WorkflowManager getWorkflowManager() {
MidPointApplication application = (MidPointApplication) MidPointApplication.get();
MidPointApplication application = MidPointApplication.get();
return application.getWorkflowManager();
}

Expand Down Expand Up @@ -193,27 +197,30 @@ protected boolean checkOrderingSettings() {
return false;
}

public boolean isOrderingDisabled() {
if (!checkOrderingSettings()) {
return false;
}
public boolean isDistinct() {
GuiObjectListType def = WebComponentUtil.getDefaultGuiObjectListType((PageBase) component.getPage());
return def == null || def.getDistinct() != DistinctSearchOptionType.NEVER; // change after other options are added
}

PageBase page = (PageBase) component.getPage();
AdminGuiConfigurationType config = page.getPrincipal().getAdminGuiConfiguration();
if (config == null) {
return false;
}
GuiObjectListsType lists = config.getObjectLists();
if (lists == null) {
return false;
protected Collection<SelectorOptions<GetOperationOptions>> createDefaultOptions() {
return getDistinctRelatedOptions(); // probably others in the future
}

@Nullable
protected Collection<SelectorOptions<GetOperationOptions>> getDistinctRelatedOptions() {
if (isDistinct()) {
return SelectorOptions.createCollection(GetOperationOptions.createDistinct());
} else {
return null;
}
}

GuiObjectListType def = lists.getDefault();
if (def == null) {
public boolean isOrderingDisabled() {
if (!checkOrderingSettings()) {
return false;
}

return def.isDisableSorting();
GuiObjectListType def = WebComponentUtil.getDefaultGuiObjectListType((PageBase) component.getPage());
return def != null && def.isDisableSorting();
}

protected ObjectPaging createPaging(long offset, long pageSize) {
Expand Down
10 changes: 8 additions & 2 deletions gui/admin-gui/src/main/java/com/evolveum/midpoint/web/component/data/ObjectDataProvider.java
View file
Expand Up @@ -78,6 +78,12 @@ public List<T> getSelectedData() {
return allSelected;
}

// Here we apply the distinct option. It is easier and more reliable to apply it here than to do at all the places
// where options for this provider are defined.
private Collection<SelectorOptions<GetOperationOptions>> getOptionsToUse() {
return GetOperationOptions.merge(options, getDistinctRelatedOptions());
}


@Override
public Iterator<W> internalIterator(long first, long count) {
Expand Down Expand Up @@ -120,7 +126,7 @@ public Iterator<W> internalIterator(long first, long count) {
LOGGER.trace("Query {} with {}", type.getSimpleName(), query.debugDump());
}

List<PrismObject<T>> list = getModel().searchObjects(type, query, options, task, result);
List<PrismObject<T>> list = getModel().searchObjects(type, query, getOptionsToUse(), task, result);

if (LOGGER.isTraceEnabled()) {
LOGGER.trace("Query {} resulted in {} objects", type.getSimpleName(), list.size());
Expand Down Expand Up @@ -169,7 +175,7 @@ protected int internalSize() {
OperationResult result = new OperationResult(OPERATION_COUNT_OBJECTS);
try {
Task task = getPage().createSimpleTask(OPERATION_COUNT_OBJECTS);
count = getModel().countObjects(type, getQuery(), options, task, result);
count = getModel().countObjects(type, getQuery(), getOptionsToUse(), task, result);
} catch (Exception ex) {
result.recordFatalError("Couldn't count objects.", ex);
LoggingUtils.logUnexpectedException(LOGGER, "Couldn't count objects", ex);
Expand Down
23 changes: 13 additions & 10 deletions .../src/main/java/com/evolveum/midpoint/web/component/data/RepositoryObjectDataProvider.java
View file
Expand Up @@ -38,6 +38,7 @@
import com.evolveum.midpoint.xml.ns._public.common.common_3.ShadowType;
import org.apache.commons.lang.Validate;
import org.apache.wicket.Component;
import org.jetbrains.annotations.NotNull;

import java.io.Serializable;
import java.util.*;
Expand Down Expand Up @@ -78,12 +79,7 @@ public Iterator<DebugObjectItem> internalIterator(long first, long count) {
}
query.setPaging(paging);

//RAW and DEFAULT retrieve option selected
Collection<SelectorOptions<GetOperationOptions>> options = new ArrayList<>();
GetOperationOptions opt = GetOperationOptions.createRaw();
opt.setRetrieve(RetrieveOption.DEFAULT);
options.add(SelectorOptions.create(ItemPath.EMPTY_PATH, opt));

Collection<SelectorOptions<GetOperationOptions>> options = getOptions();
List<PrismObject<? extends ObjectType>> list = getModel().searchObjects((Class) type, query, options,
getPage().createSimpleTask(OPERATION_SEARCH_OBJECTS), result);
for (PrismObject<? extends ObjectType> object : list) {
Expand All @@ -101,6 +97,15 @@ public Iterator<DebugObjectItem> internalIterator(long first, long count) {
return getAvailableData().iterator();
}

@NotNull
private Collection<SelectorOptions<GetOperationOptions>> getOptions() {
Collection<SelectorOptions<GetOperationOptions>> options = new ArrayList<>();
GetOperationOptions opt = GetOperationOptions.createRaw();
opt.setRetrieve(RetrieveOption.DEFAULT);
options.add(SelectorOptions.create(ItemPath.EMPTY_PATH, opt));
return GetOperationOptions.merge(createDefaultOptions(), options);
}

@Override
protected boolean checkOrderingSettings() {
return true;
Expand Down Expand Up @@ -171,16 +176,14 @@ protected int internalSize() {
int count = 0;
OperationResult result = new OperationResult(OPERATION_COUNT_OBJECTS);
try {
count = getModel().countObjects(type, getQuery(),
SelectorOptions.createCollection(ItemPath.EMPTY_PATH, GetOperationOptions.createRaw()),
count = getModel().countObjects(type, getQuery(), getOptions(),
getPage().createSimpleTask(OPERATION_COUNT_OBJECTS), result);
} catch (Exception ex) {
result.recordFatalError("Couldn't count objects.", ex);
} finally {
result.computeStatusIfUnknown();
}

getPage().showResult(result, false);
getPage().showResult(result, false);
LOGGER.trace("end::internalSize()");
return count;
}
Expand Down
6 changes: 4 additions & 2 deletions .../main/java/com/evolveum/midpoint/web/component/data/SelectableBeanObjectDataProvider.java
View file
Expand Up @@ -161,7 +161,7 @@ public Iterator<SelectableBean<O>> internalIterator(long offset, long pageSize)
LOGGER.trace("Query {} with {}", type.getSimpleName(), query.debugDump());
}

if (ResourceType.class.equals(type) && (options == null || options.isEmpty())){
if (ResourceType.class.equals(type) && (options == null || options.isEmpty())) {
options = SelectorOptions.createCollection(GetOperationOptions.createNoFetch());
}
Collection<SelectorOptions<GetOperationOptions>> currentOptions = options;
Expand All @@ -178,6 +178,7 @@ public Iterator<SelectableBean<O>> internalIterator(long offset, long pageSize)
(o) -> o.setDefinitionProcessing(FULL));
}
}
currentOptions = GetOperationOptions.merge(currentOptions, getDistinctRelatedOptions());
List<PrismObject<? extends O>> list = (List)getModel().searchObjects(type, query, currentOptions, task, result);

if (LOGGER.isTraceEnabled()) {
Expand Down Expand Up @@ -240,7 +241,8 @@ protected int internalSize() {
Task task = getPage().createSimpleTask(OPERATION_COUNT_OBJECTS);
OperationResult result = task.getResult();
try {
Integer counted = getModel().countObjects(type, getQuery(), options, task, result);
Collection<SelectorOptions<GetOperationOptions>> currentOptions = GetOperationOptions.merge(options, getDistinctRelatedOptions());
Integer counted = getModel().countObjects(type, getQuery(), currentOptions, task, result);
count = defaultIfNull(counted, 0);
} catch (Exception ex) {
result.recordFatalError("Couldn't count objects.", ex);
Expand Down
6 changes: 3 additions & 3 deletions gui/admin-gui/src/main/java/com/evolveum/midpoint/web/component/search/SearchFactory.java
View file
Expand Up @@ -217,11 +217,11 @@ private static <T extends ObjectType> boolean isFullTextSearchEnabled(ModelServi
return FullTextSearchConfigurationUtil.isEnabledFor(modelServiceLocator.getModelInteractionService().getSystemConfiguration(result)
.getFullTextSearch(), type);
} catch (SchemaException | ObjectNotFoundException ex) {
throw new SystemException(ex);
throw new SystemException(ex);
}
}

private static <T extends ObjectType> SearchBoxModeType getDefaultSearchType (ModelServiceLocator modelServiceLocator, Class<T> type) {
private static <T extends ObjectType> SearchBoxModeType getDefaultSearchType(ModelServiceLocator modelServiceLocator, Class<T> type) {
OperationResult result = new OperationResult(LOAD_ADMIN_GUI_CONFIGURATION);
try {
AdminGuiConfigurationType guiConfig = modelServiceLocator.getModelInteractionService().getAdminGuiConfiguration(null, result);
Expand All @@ -238,7 +238,7 @@ private static <T extends ObjectType> SearchBoxModeType getDefaultSearchType (Mo
}
return null;
} catch (SchemaException | ObjectNotFoundException ex) {
throw new SystemException(ex);
throw new SystemException(ex);
}
}

Expand Down
4 changes: 2 additions & 2 deletions ...in-gui/src/main/java/com/evolveum/midpoint/web/page/admin/server/dto/NodeDtoProvider.java
View file
Expand Up @@ -60,7 +60,7 @@ public Iterator<? extends NodeDto> internalIterator(long first, long count) {
}
query.setPaging(paging);

List<PrismObject<NodeType>> nodes = getModel().searchObjects(NodeType.class, query, null, task, result);
List<PrismObject<NodeType>> nodes = getModel().searchObjects(NodeType.class, query, createDefaultOptions(), task, result);

for (PrismObject<NodeType> node : nodes) {
getAvailableData().add(createNodeDto(node));
Expand Down Expand Up @@ -104,7 +104,7 @@ protected int internalSize() {
OperationResult result = new OperationResult(OPERATION_COUNT_NODES);
Task task = getTaskManager().createTaskInstance(OPERATION_COUNT_NODES);
try {
count = getModel().countObjects(NodeType.class, getQuery(), null, task, result);
count = getModel().countObjects(NodeType.class, getQuery(), createDefaultOptions(), task, result);
result.recomputeStatus();
} catch (Exception ex) {
LoggingUtils.logUnexpectedException(LOGGER, "Unhandled exception when counting nodes", ex);
Expand Down
7 changes: 4 additions & 3 deletions ...in-gui/src/main/java/com/evolveum/midpoint/web/page/admin/server/dto/TaskDtoProvider.java
View file
Expand Up @@ -84,8 +84,9 @@ public Iterator<? extends TaskDto> internalIterator(long first, long count) {
propertiesToGet.add(TaskType.F_NEXT_RUN_START_TIMESTAMP);
propertiesToGet.add(TaskType.F_NEXT_RETRY_TIMESTAMP);
}
Collection<SelectorOptions<GetOperationOptions>> searchOptions =
GetOperationOptions.createRetrieveAttributesOptions(propertiesToGet.toArray(new QName[0]));
Collection<SelectorOptions<GetOperationOptions>> searchOptions = GetOperationOptions.merge(
createDefaultOptions(),
GetOperationOptions.createRetrieveAttributesOptions(propertiesToGet.toArray(new QName[0])));
List<PrismObject<TaskType>> tasks = getModel().searchObjects(TaskType.class, query, searchOptions, operationTask, result);
for (PrismObject<TaskType> task : tasks) {
try {
Expand Down Expand Up @@ -140,7 +141,7 @@ protected int internalSize() {
OperationResult result = new OperationResult(OPERATION_COUNT_TASKS);
Task task = getTaskManager().createTaskInstance(OPERATION_COUNT_TASKS);
try {
count = getModel().countObjects(TaskType.class, getQuery(), null, task, result);
count = getModel().countObjects(TaskType.class, getQuery(), createDefaultOptions(), task, result);
result.recomputeStatus();
} catch (Exception ex) {
LoggingUtils.logUnexpectedException(LOGGER, "Unhandled exception when counting tasks", ex);
Expand Down

0 comments on commit cd3cbcb

Please sign in to comment.