Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/feature/smart-correlation-protot…
Browse files Browse the repository at this point in the history 
…ype' into feature/smart-correlation-prototype
  • Loading branch information
@mederly
mederly committed Aug 4, 2022
2 parents 51efc9f + 0c47865 commit 2d2531d
Show file tree
Hide file tree
Showing 13 changed files with 159 additions and 138 deletions.
107 changes: 72 additions & 35 deletions infra/schema/src/main/java/com/evolveum/midpoint/schema/SelectorOptions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@

import com.evolveum.midpoint.prism.PrismContext;
import com.evolveum.midpoint.prism.path.ItemPath;
import com.evolveum.midpoint.prism.path.ItemPathCollectionsUtil;
import com.evolveum.midpoint.prism.path.ItemPathComparatorUtil;
import com.evolveum.midpoint.prism.path.UniformItemPath;
import com.evolveum.midpoint.prism.util.CloneUtil;
import com.evolveum.midpoint.util.DebugDumpable;
Expand Down Expand Up @@ -122,7 +122,8 @@ public static <T> T findRootOptions(Collection<SelectorOptions<T>> options) {
}

// newValueSupplier should be not null as well (unless the caller is 100% sure it won't be needed)
public static <T> @NotNull Collection<SelectorOptions<T>> updateRootOptions(@Nullable Collection<SelectorOptions<T>> options,
public static <T> @NotNull Collection<SelectorOptions<T>> updateRootOptions(
@Nullable Collection<SelectorOptions<T>> options,
@NotNull Consumer<T> updater, Supplier<T> newValueSupplier) {
if (options == null) {
options = new ArrayList<>();
Expand Down Expand Up @@ -156,36 +157,32 @@ public boolean isRoot() {
return itemPathOrNull == null || itemPathOrNull.isEmpty();
}

// TODO find a better way to specify this
private static final Set<ItemPath> PATHS_NOT_RETURNED_BY_DEFAULT = new HashSet<>(Arrays.asList(
ItemPath.create(FocusType.F_JPEG_PHOTO),
ItemPath.create(FocusType.F_IDENTITIES),
ItemPath.create(TaskType.F_RESULT),
ItemPath.create(TaskType.F_SUBTASK_REF),
ItemPath.create(TaskType.F_NODE_AS_OBSERVED),
ItemPath.create(TaskType.F_NEXT_RUN_START_TIMESTAMP),
ItemPath.create(TaskType.F_NEXT_RETRY_TIMESTAMP),
ItemPath.create(LookupTableType.F_ROW),
ItemPath.create(AccessCertificationCampaignType.F_CASE)));

private static final Set<Class<?>> OBJECTS_NOT_RETURNED_FULLY_BY_DEFAULT = new HashSet<>(Arrays.asList(
UserType.class, RoleType.class, OrgType.class, ServiceType.class, AbstractRoleType.class,
FocusType.class, AssignmentHolderType.class, ObjectType.class,
TaskType.class, LookupTableType.class, AccessCertificationCampaignType.class,
ShadowType.class // because of index-only attributes
ShadowType.class // because of index-only attributes
));

public static boolean isRetrievedFullyByDefault(Class<?> objectType) {
return !OBJECTS_NOT_RETURNED_FULLY_BY_DEFAULT.contains(objectType);
}

public static boolean hasToLoadPath(
@NotNull ItemPath path,
@Nullable Collection<SelectorOptions<GetOperationOptions>> options) {
return hasToLoadPath(path, options, !ItemPathCollectionsUtil.containsEquivalent(PATHS_NOT_RETURNED_BY_DEFAULT, path));
}

public static boolean hasToLoadPath(
/**
* Returns true if the asked path must be included in the object based on the provided get options.
* See {@link GetOperationOptions#retrieve} javadoc for more details.
* Retrieve option with null or empty path asks to load everything.
*
* This is not used only in repository but also for retrieving information during provisioning.
* Default value must be explicitly provided as it depends on the context of the question.
*
* Simplified example:
* When we ask for path "x/y", the path itself and anything under it is to be included.
* User wants "x/y", as well as "x/y/ya", etc.
* User does not say anything about path "x" (prefix) or "z" (different path altogether).
* If no option defines what to do with the asked path, defaultValue is returned.
*/
public static boolean hasToIncludePath(
@NotNull ItemPath path,
@Nullable Collection<SelectorOptions<GetOperationOptions>> options,
boolean defaultValue) {
Expand All @@ -195,22 +192,11 @@ public static boolean hasToLoadPath(

for (SelectorOptions<GetOperationOptions> option : emptyIfNull(options)) {
// TODO consider ordering of the options from most specific to least specific
// (currently not mandated by GetOperationOptions.retrieve specification).
RetrieveOption retrievalCommand = option != null
&& option.getOptions() != null ? option.getOptions().getRetrieve() : null;
if (retrievalCommand != null) {
ObjectSelector selector = option.getSelector();
/*
* TODO consult with Palo/Tony:
* This has confusing semantics, because when asked whether F_CASE needs to be loaded when
* retrieve option has path "case/5", it returns false (or default value).
* That indicates "no, we don't need to load cases".
* While it's a fact that not all cases needs to be loaded, we also can't just skip all the cases either.
* So there is a virtually identical method like this in QAccessCertificationCampaignMapping, but with
* reversed isSubPathOrEquivalent condition (commented version).
* But with the reversed version, the options with "" path don't work (that probably means "fetch all").
* That fails SqaleRepoSmokeTest#test222PhotoPersistenceReindex.
*/
// if (selector == null || selector.getPath() == null || path.isSubPathOrEquivalent(selector.getPath())) {
if (selector == null || selector.getPath() == null || selector.getPath().isSubPathOrEquivalent(path)) {
switch (retrievalCommand) {
case EXCLUDE:
Expand All @@ -228,14 +214,65 @@ public static boolean hasToLoadPath(
return defaultValue;
}

/**
* Returns true if the asked path must be considered for fetching based on the provided get options.
* This is different from {@link #hasToIncludePath(ItemPath, Collection, boolean)} as it considers
* path relation in any direction (subpath or prefix or equal).
* This is typical for repository usage that asks questions like:
* "Should I load container x *that I don't fetch by default*?"
* When the user specifies options with retrieve of the path inside that container, e.g. "x/a"),
* he does not state that he wants to fetch "x", but the repo can't just ignore processing of "x"
* and skip it altogether if it is stored separately.
* At the same time, original semantics of {@link #hasToIncludePath(ItemPath, Collection, boolean)}
* has to hold as well - if user asks to load "a", repo needs to fetch "a/b" as well (everything under "a").
*
* Simplified example:
* When we ask for path "x/y", the path itself, any subpath and any prefix must be considered for fetching.
* User wants "x/y", as well as "x/y/ya", etc.
* User does not say anything about path "x" (prefix) or "z" (different path altogether),
* but while we can ignore "z", we must consider "x" if the repo does not provide it by default.
* In that case it needs to fetch it (fully or partially, whatever is practical and/or efficient)
* to provide the requested "x/y" content.
*/
public static boolean hasToFetchPathNotRetrievedByDefault(
@NotNull ItemPath path,
@Nullable Collection<SelectorOptions<GetOperationOptions>> options) {
if (options == null) {
return false;
}

for (SelectorOptions<GetOperationOptions> option : emptyIfNull(options)) {
RetrieveOption retrievalCommand = option != null
&& option.getOptions() != null ? option.getOptions().getRetrieve() : null;
if (retrievalCommand != null) {
ObjectSelector selector = option.getSelector();
if (selector == null || selector.getPath() == null
// shortcut for "is subpath or prefix or equal"
|| ItemPathComparatorUtil.compareComplex(selector.getPath(), path) != ItemPath.CompareResult.NO_RELATION) {
switch (retrievalCommand) {
case EXCLUDE:
case DEFAULT:
return false;
case INCLUDE:
return true;
default:
throw new AssertionError("Wrong retrieve option: " + retrievalCommand);
}
}
}
}
return false;
}

public static List<SelectorOptions<GetOperationOptions>> filterRetrieveOptions(
Collection<SelectorOptions<GetOperationOptions>> options) {
return MiscUtil.streamOf(options)
.filter(option -> option.getOptions() != null && option.getOptions().getRetrieve() != null)
.collect(Collectors.toList());
}

public static <T> Map<T, Collection<UniformItemPath>> extractOptionValues(Collection<SelectorOptions<GetOperationOptions>> options,
public static <T> Map<T, Collection<UniformItemPath>> extractOptionValues(
Collection<SelectorOptions<GetOperationOptions>> options,
Function<GetOperationOptions, T> supplier, PrismContext prismContext) {
Map<T, Collection<UniformItemPath>> rv = new HashMap<>();
final UniformItemPath emptyPath = prismContext.emptyPath();
Expand Down
45 changes: 23 additions & 22 deletions ...oning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/ProvisioningContext.java
View file
Original file line number Diff line number Diff line change
@@ -1,26 +1,34 @@
/*
* Copyright (c) 2015-2017 Evolveum and contributors
* Copyright (C) 2015-2022 Evolveum and contributors
*
* This work is dual-licensed under the Apache License 2.0
* and European Union Public License. See LICENSE file for details.
*/
package com.evolveum.midpoint.provisioning.impl;

import com.evolveum.midpoint.provisioning.impl.resources.ResourceManager;
import com.evolveum.midpoint.provisioning.ucf.api.AttributesToReturn;
import com.evolveum.midpoint.schema.processor.*;
import java.util.*;
import java.util.function.Supplier;
import javax.xml.namespace.QName;

import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;

import com.evolveum.midpoint.prism.PrismContext;
import com.evolveum.midpoint.prism.PrismObject;
import com.evolveum.midpoint.prism.path.ItemPath;
import com.evolveum.midpoint.prism.query.ObjectFilter;
import com.evolveum.midpoint.provisioning.impl.resources.ResourceManager;
import com.evolveum.midpoint.provisioning.ucf.api.AttributesToReturn;
import com.evolveum.midpoint.provisioning.ucf.api.ConnectorInstance;
import com.evolveum.midpoint.provisioning.ucf.api.UcfExecutionContext;
import com.evolveum.midpoint.provisioning.util.ProvisioningUtil;
import com.evolveum.midpoint.repo.common.expression.ExpressionFactory;
import com.evolveum.midpoint.repo.common.expression.ExpressionUtil;
import com.evolveum.midpoint.schema.*;
import com.evolveum.midpoint.schema.expression.VariablesMap;
import com.evolveum.midpoint.schema.GetOperationOptions;
import com.evolveum.midpoint.schema.SelectorOptions;
import com.evolveum.midpoint.schema.constants.ExpressionConstants;
import com.evolveum.midpoint.schema.expression.VariablesMap;
import com.evolveum.midpoint.schema.processor.*;
import com.evolveum.midpoint.schema.result.OperationResult;
import com.evolveum.midpoint.schema.util.MiscSchemaUtil;
import com.evolveum.midpoint.schema.util.ResourceTypeUtil;
Expand All @@ -34,14 +42,6 @@
import com.evolveum.midpoint.xml.ns._public.common.common_3.*;
import com.evolveum.midpoint.xml.ns._public.resource.capabilities_3.CapabilityType;

import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;

import javax.xml.namespace.QName;
import java.util.*;
import java.util.Objects;
import java.util.function.Supplier;

/**
* Context for provisioning operations. Contains key information like resolved resource,
* object type and/or object class definitions, and so on.
Expand Down Expand Up @@ -167,7 +167,8 @@ public void setPropagation(boolean value) {
this.propagation = value;
}

@NotNull public ResourceType getResource() {
@NotNull
public ResourceType getResource() {
return resource;
}

Expand Down Expand Up @@ -391,14 +392,14 @@ private <T extends CapabilityType> ConnectorInstance getConnectorInstance(
return contextFactory.getResourceManager()
.getConfiguredConnectorInstance(resource.asPrismObject(), operationCapabilityClass, false, result);
} catch (ObjectNotFoundException | SchemaException e) {
result.recordPartialError("Could not get connector instance " + getDesc() + ": " + e.getMessage(), e);
result.recordPartialError("Could not get connector instance " + getDesc() + ": " + e.getMessage(), e);
// Wrap those exceptions to a configuration exception. In the context of the provisioning operation we really cannot throw
// ObjectNotFoundException exception. If we do that then the consistency code will interpret that as if the resource object
// (shadow) is missing. But that's wrong. We do not have connector therefore we do not know anything about the shadow. We cannot
// throw ObjectNotFoundException here.
throw new ConfigurationException(e.getMessage(), e);
} catch (CommunicationException | ConfigurationException | RuntimeException e) {
result.recordPartialError("Could not get connector instance " + getDesc() + ": " + e.getMessage(), e);
result.recordPartialError("Could not get connector instance " + getDesc() + ": " + e.getMessage(), e);
throw e;
} finally {
result.close();
Expand All @@ -421,7 +422,7 @@ public <T extends CapabilityType> T getCapability(@NotNull Class<T> capabilityCl

@Override
public String toString() {
return "ProvisioningContext("+getDesc()+")";
return "ProvisioningContext(" + getDesc() + ")";
}

public ItemPath path(Object... components) {
Expand Down Expand Up @@ -481,8 +482,8 @@ public boolean isTypeBased() {
}

public void validateSchema(ShadowType shadow)
throws ObjectNotFoundException,
SchemaException, CommunicationException, ConfigurationException, ExpressionEvaluationException {
throws ObjectNotFoundException,
SchemaException, CommunicationException, ConfigurationException, ExpressionEvaluationException {
if (ResourceTypeUtil.isValidateSchema(resource)) {
ShadowUtil.validateAttributeSchema(shadow, getObjectDefinition());
}
Expand All @@ -493,11 +494,11 @@ public void validateSchema(ShadowType shadow)
}

public boolean isFetchingRequested(ItemPath path) {
return SelectorOptions.hasToLoadPath(path, getOperationOptions, false);
return SelectorOptions.hasToIncludePath(path, getOperationOptions, false);
}

public boolean isFetchingNotDisabled(ItemPath path) {
return SelectorOptions.hasToLoadPath(path, getOperationOptions, true);
return SelectorOptions.hasToIncludePath(path, getOperationOptions, true);
}

/**
Expand Down
22 changes: 11 additions & 11 deletions ...ning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/shadows/SearchHelper.java
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,9 @@
/*
* Copyright (C) 2010-2021 Evolveum and contributors
* Copyright (C) 2010-2022 Evolveum and contributors
*
* This work is dual-licensed under the Apache License 2.0
* and European Union Public License. See LICENSE file for details.
*/

package com.evolveum.midpoint.provisioning.impl.shadows;

import static com.evolveum.midpoint.schema.GetOperationOptions.getErrorReportingMethod;
Expand All @@ -16,11 +15,6 @@
import java.util.concurrent.atomic.AtomicInteger;
import javax.xml.namespace.QName;

import com.evolveum.midpoint.provisioning.util.DefinitionsUtil;
import com.evolveum.midpoint.schema.processor.ResourceObjectDefinition;

import com.evolveum.midpoint.schema.util.ObjectQueryUtil;

import org.jetbrains.annotations.NotNull;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
Expand All @@ -32,27 +26,33 @@
import com.evolveum.midpoint.prism.query.*;
import com.evolveum.midpoint.provisioning.impl.ProvisioningContext;
import com.evolveum.midpoint.provisioning.impl.ProvisioningContextFactory;
import com.evolveum.midpoint.provisioning.impl.resourceobjects.ResourceObjectConverter;
import com.evolveum.midpoint.provisioning.impl.ShadowCaretaker;
import com.evolveum.midpoint.provisioning.impl.resourceobjects.ResourceObjectConverter;
import com.evolveum.midpoint.provisioning.impl.resourceobjects.ResourceObjectFound;
import com.evolveum.midpoint.provisioning.impl.resourceobjects.ResourceObjectHandler;
import com.evolveum.midpoint.provisioning.impl.shadows.manager.ShadowManager;
import com.evolveum.midpoint.provisioning.ucf.api.ConnectorInstance;
import com.evolveum.midpoint.provisioning.ucf.api.GenericFrameworkException;
import com.evolveum.midpoint.provisioning.util.DefinitionsUtil;
import com.evolveum.midpoint.provisioning.util.ProvisioningUtil;
import com.evolveum.midpoint.repo.common.expression.ExpressionFactory;
import com.evolveum.midpoint.schema.*;
import com.evolveum.midpoint.schema.internals.InternalCounters;
import com.evolveum.midpoint.schema.internals.InternalMonitor;
import com.evolveum.midpoint.schema.internals.InternalsConfig;
import com.evolveum.midpoint.schema.processor.ResourceObjectDefinition;
import com.evolveum.midpoint.schema.result.OperationResult;
import com.evolveum.midpoint.schema.util.ObjectQueryUtil;
import com.evolveum.midpoint.task.api.Task;
import com.evolveum.midpoint.util.QNameUtil;
import com.evolveum.midpoint.util.annotation.Experimental;
import com.evolveum.midpoint.util.exception.*;
import com.evolveum.midpoint.util.logging.Trace;
import com.evolveum.midpoint.util.logging.TraceManager;
import com.evolveum.midpoint.xml.ns._public.common.common_3.*;
import com.evolveum.midpoint.xml.ns._public.common.common_3.CachingMetadataType;
import com.evolveum.midpoint.xml.ns._public.common.common_3.FetchErrorReportingMethodType;
import com.evolveum.midpoint.xml.ns._public.common.common_3.ResourceType;
import com.evolveum.midpoint.xml.ns._public.common.common_3.ShadowType;
import com.evolveum.midpoint.xml.ns._public.resource.capabilities_3.CountObjectsCapabilityType;
import com.evolveum.midpoint.xml.ns._public.resource.capabilities_3.CountObjectsSimulateType;
import com.evolveum.midpoint.xml.ns._public.resource.capabilities_3.ReadCapabilityType;
Expand Down Expand Up @@ -156,7 +156,7 @@ private SearchResultMetadata searchObjectIterativeResource(
};

ObjectQuery attributeQuery = createAttributeQuery(query);
boolean fetchAssociations = SelectorOptions.hasToLoadPath(ShadowType.F_ASSOCIATION, options);
boolean fetchAssociations = SelectorOptions.hasToIncludePath(ShadowType.F_ASSOCIATION, options, true);
try {
return resourceObjectConverter.searchResourceObjects(
ctx, resultHandler, attributeQuery, fetchAssociations, ucfErrorReportingMethod, parentResult);
Expand Down Expand Up @@ -249,7 +249,7 @@ private List<ObjectFilter> createAttributeQueryInternal(List<? extends ObjectFil
} else if (f instanceof SubstringFilter) { // TODO fix
attributeFilter.add(f);
} else if (f instanceof RefFilter) {
ItemPath parentPath = ((RefFilter)f).getParentPath();
ItemPath parentPath = ((RefFilter) f).getParentPath();
if (parentPath.isEmpty()) {
QName elementName = ((RefFilter) f).getElementName();
if (QNameUtil.match(ShadowType.F_RESOURCE_REF, elementName)) {
Expand Down

0 comments on commit 2d2531d

Please sign in to comment.