Add on-demand fetching of items for mappings

This commit resolves MID-5772: When a mapping source is incomplete
(e.g. $immediateRole/jpegPhoto) we load it on demand.

Note that the solution provided here is not 100% finished, from both
functional (MID-6001) and performance (MID-6002) point of view.

This commit also fixes a fault in RepositoryCache where we considered
non-user focus types (e.g. ServiceType, RoleType, OrgType) as loaded
fully even with no RETRIEVE options specified. This is not true
because they can contain jpegPhoto! Fixing this bug can result in
some performance degradation because cache is avoided in more cases
than before. See also MID-6003.

Other minor changes:
- SimpleTaskAdapter was renamed to NullTaskImpl and moved to task-api.
- Minor refactorings were done.

infra/prism-api/src/main/java/com/evolveum/midpoint/prism/util/ItemDeltaItem.java

@@ -49,13 +49,13 @@ public class ItemDeltaItem<V extends PrismValue,D extends ItemDefinition> implem
     private D definition;
     private ItemPath resolvePath = ItemPath.EMPTY_PATH;
 
-    // Residual path is a temporary solution to Structured attriebutes in 3.x and 4.x.
+    // Residual path is a temporary solution to Structured attributes in 3.x and 4.x.
     // It should disappear in 5.x.
     private ItemPath residualPath = null;
 
     // The deltas in sub-items. E.g. if this object represents "ContainerDeltaContainer"
     // this property contains property deltas that may exist inside the container.
-    Collection<? extends ItemDelta<?,?>> subItemDeltas;
+    private Collection<? extends ItemDelta<?,?>> subItemDeltas;
 
     // For clone and ObjectDeltaObject
     protected ItemDeltaItem() { }

infra/schema/src/main/java/com/evolveum/midpoint/schema/SelectorOptions.java

@@ -181,7 +181,8 @@ public boolean isRoot() {
             ItemPath.create(AccessCertificationCampaignType.F_CASE)));
 
     private static final Set<Class<?>> OBJECTS_NOT_RETURNED_FULLY_BY_DEFAULT = new HashSet<>(Arrays.asList(
-            UserType.class, FocusType.class, AssignmentHolderType.class, ObjectType.class,
+            UserType.class, RoleType.class, OrgType.class, ServiceType.class, AbstractRoleType.class,
+            FocusType.class, AssignmentHolderType.class, ObjectType.class,
             TaskType.class, LookupTableType.class, AccessCertificationCampaignType.class,
             ShadowType.class            // because of index-only attributes
     ));

infra/util/src/main/java/com/evolveum/midpoint/util/PrettyPrinter.java

@@ -251,19 +251,19 @@ public static String prettyPrint(Object value) {
         if (value == null) {
             return "null";
         }
-        String out = null;
         if (value instanceof JAXBElement) {
             Object elementValue = ((JAXBElement)value).getValue();
-            out = tryPrettyPrint(elementValue);
-            if (out != null) {
-                return ("JAXBElement("+((JAXBElement)value).getName()+","+out+")");
+            String attempt = tryPrettyPrint(elementValue);
+            if (attempt != null) {
+                return ("JAXBElement("+((JAXBElement)value).getName()+","+attempt+")");
             }
         }
-        out = tryPrettyPrint(value);
-        if (out == null) {
-            out = value.toString();
+        String attempt = tryPrettyPrint(value);
+        if (attempt != null) {
+            return attempt;
+        } else {
+            return value.toString();
         }
-        return out;
     }
 
     private static String tryPrettyPrint(Object value) {

model/model-common/src/main/java/com/evolveum/midpoint/model/common/mapping/MappingFactory.java

@@ -77,12 +77,14 @@ public <V extends PrismValue, D extends ItemDefinition> MappingImpl.Builder<V, D
                 .expressionFactory(expressionFactory)
                 .securityContextManager(securityContextManager)
                 .variables(new ExpressionVariables())
+                .objectResolver(objectResolver)
                 .profiling(profiling);
     }
 
     public <V extends PrismValue, D extends ItemDefinition> MappingImpl.Builder<V, D> createMappingBuilder(MappingType mappingType, String shortDesc) {
         return this.<V,D>createMappingBuilder().mappingType(mappingType)
-                .contextDescription(shortDesc);
+                .contextDescription(shortDesc)
+                .objectResolver(objectResolver);
     }
 
 }

model/model-common/src/test/java/com/evolveum/midpoint/model/common/AbstractModelCommonTest.java

@@ -8,6 +8,8 @@
 
 import java.io.File;
 
+import com.evolveum.midpoint.task.api.Task;
+import com.evolveum.midpoint.task.api.test.NullTaskImpl;
 import com.evolveum.midpoint.test.util.TestUtil;
 
 /**
@@ -33,4 +35,7 @@ protected void displayThen(final String TEST_NAME) {
         TestUtil.displayThen(TEST_NAME);
     }
 
+    protected Task createTask() {
+        return new NullTaskImpl();
+    }
 }

model/model-common/src/test/java/com/evolveum/midpoint/model/common/expression/TestExpression.java

@@ -397,7 +397,7 @@ public void test200IterationCondition() throws Exception {
         variables.put(ExpressionConstants.VAR_ITERATION_TOKEN, "001",
                 TestUtil.createPrimitivePropertyDefinition(prismContext, ExpressionConstants.VAR_ITERATION_TOKEN, PrimitiveType.STRING));
 
-        ExpressionEvaluationContext expressionContext = new ExpressionEvaluationContext(null , variables, TEST_NAME, null);
+        ExpressionEvaluationContext expressionContext = new ExpressionEvaluationContext(null , variables, TEST_NAME, createTask());
 
         // WHEN
         PrismValueDeltaSetTriple<PrismPropertyValue<Boolean>> outputTriple =
@@ -445,7 +445,7 @@ protected <V extends PrismValue, D extends ItemDefinition> PrismValueDeltaSetTri
             ExpressionType expressionType, D outputDefinition, ExpressionEvaluationContext expressionContext,
             OperationResult result)
                     throws SchemaException, ObjectNotFoundException, SecurityViolationException, ExpressionEvaluationException, CommunicationException, ConfigurationException {
-        Expression<V,D> expression = expressionFactory.makeExpression(expressionType, outputDefinition , getExpressionProfile(),
+        Expression<V,D> expression = expressionFactory.makeExpression(expressionType, outputDefinition, getExpressionProfile(),
                 expressionContext.getContextDescription(), expressionContext.getTask(), result);
         LOGGER.debug("Starting evaluation of expression: {}", expression);
         return expression.evaluate(expressionContext, result);

model/model-common/src/test/java/com/evolveum/midpoint/model/common/expression/TestExpressionUtil.java

@@ -15,6 +15,9 @@
 import com.evolveum.midpoint.prism.util.ObjectDeltaObject;
 import com.evolveum.midpoint.prism.util.PrismAsserts;
 
+import com.evolveum.midpoint.repo.common.DirectoryFileObjectResolver;
+import com.evolveum.midpoint.repo.common.ObjectResolver;
+import com.evolveum.midpoint.task.api.test.NullTaskImpl;
 import org.testng.annotations.Test;
 
 import java.io.File;
@@ -200,12 +203,14 @@ private <T> T resolvePath(String path, ExpressionVariables variables, final Stri
         ItemPath itemPath = toItemPath(path);
 
         // WHEN
-        Object resolved = ExpressionUtil.resolvePathGetValue(itemPath, variables, false, null, null,
-                PrismTestUtil.getPrismContext(), TEST_NAME, null, result);
+        ObjectResolver objectResolver = new DirectoryFileObjectResolver(MidPointTestConstants.OBJECTS_DIR);
+        Object resolved = ExpressionUtil.resolvePathGetValue(itemPath, variables, false, null, objectResolver,
+                PrismTestUtil.getPrismContext(), TEST_NAME, new NullTaskImpl(), result);
 
         // THEN
         IntegrationTestTools.display("Resolved", resolved);
 
+        //noinspection unchecked
         return (T) resolved;
     }
 

model/model-common/src/test/java/com/evolveum/midpoint/model/common/mapping/MappingTestEvaluator.java

@@ -19,6 +19,7 @@
 import com.evolveum.midpoint.prism.delta.*;
 import com.evolveum.midpoint.prism.path.ItemName;
 import com.evolveum.midpoint.prism.path.ItemPath;
+import com.evolveum.midpoint.task.api.test.NullTaskImpl;
 import com.evolveum.prism.xml.ns._public.types_3.ProtectedStringType;
 
 import org.xml.sax.SAXException;
@@ -279,7 +280,7 @@ public <T,I> PrismValueDeltaSetTriple<PrismPropertyValue<T>> evaluateMapping(Str
             throws SchemaException, IOException, JAXBException, ExpressionEvaluationException, ObjectNotFoundException, EncryptionException, SecurityViolationException, ConfigurationException, CommunicationException {
         MappingImpl<PrismPropertyValue<T>,PrismPropertyDefinition<T>> mapping = createMapping(filename, testName, defaultTargetPropertyPath, null);
         OperationResult opResult = new OperationResult(testName);
-        mapping.evaluate(null, opResult);
+        mapping.evaluate(new NullTaskImpl(), opResult);
         assertResult(opResult);
         PrismValueDeltaSetTriple<PrismPropertyValue<T>> outputTriple = mapping.getOutputTriple();
         if (outputTriple != null) {
@@ -293,7 +294,7 @@ public <T,I> PrismValueDeltaSetTriple<PrismPropertyValue<T>> evaluateMapping(Str
             throws SchemaException, IOException, JAXBException, ExpressionEvaluationException, ObjectNotFoundException, EncryptionException, SecurityViolationException, ConfigurationException, CommunicationException {
         MappingImpl<PrismPropertyValue<T>,PrismPropertyDefinition<T>> mapping = createMapping(filename, testName, defaultTargetPropertyName, null);
         OperationResult opResult = new OperationResult(testName);
-        mapping.evaluate(null, opResult);
+        mapping.evaluate(new NullTaskImpl(), opResult);
         assertResult(opResult);
         PrismValueDeltaSetTriple<PrismPropertyValue<T>> outputTriple = mapping.getOutputTriple();
         if (outputTriple != null) {
@@ -307,7 +308,7 @@ public <T,I> PrismValueDeltaSetTriple<PrismPropertyValue<T>> evaluateMapping(Str
             throws SchemaException, IOException, JAXBException, ExpressionEvaluationException, ObjectNotFoundException, EncryptionException, SecurityViolationException, ConfigurationException, CommunicationException {
         MappingImpl<PrismPropertyValue<T>,PrismPropertyDefinition<T>> mapping = createMapping(filename, testName, defaultTargetPropertyName, null);
         OperationResult opResult = new OperationResult(testName);
-        mapping.evaluate(null, opResult);
+        mapping.evaluate(new NullTaskImpl(), opResult);
         assertResult(opResult);
         PrismValueDeltaSetTriple<PrismPropertyValue<T>> outputTriple = mapping.getOutputTriple();
         if (outputTriple != null) {
@@ -339,7 +340,7 @@ public <T,I> PrismValueDeltaSetTriple<PrismPropertyValue<T>> evaluateMappingDyna
                         valuesToAdd);
         MappingImpl<PrismPropertyValue<T>,PrismPropertyDefinition<T>> mapping = createMapping(filename, testName, defaultTargetPropertyPath, userDelta);
         OperationResult opResult = new OperationResult(testName);
-        mapping.evaluate(null, opResult);
+        mapping.evaluate(new NullTaskImpl(), opResult);
         assertResult(opResult);
         PrismValueDeltaSetTriple<PrismPropertyValue<T>> outputTriple = mapping.getOutputTriple();
         if (outputTriple != null) {
@@ -356,7 +357,7 @@ public <T,I> PrismValueDeltaSetTriple<PrismPropertyValue<T>> evaluateMappingDyna
                         valuesToAdd);
         MappingImpl<PrismPropertyValue<T>,PrismPropertyDefinition<T>> mapping = createMapping(filename, testName, defaultTargetPropertyName, userDelta);
         OperationResult opResult = new OperationResult(testName);
-        mapping.evaluate(null, opResult);
+        mapping.evaluate(new NullTaskImpl(), opResult);
         assertResult(opResult);
         PrismValueDeltaSetTriple<PrismPropertyValue<T>> outputTriple = mapping.getOutputTriple();
         if (outputTriple != null) {
@@ -373,7 +374,7 @@ public <T,I> PrismValueDeltaSetTriple<PrismPropertyValue<T>> evaluateMappingDyna
                         valuesToReplace);
         MappingImpl<PrismPropertyValue<T>,PrismPropertyDefinition<T>> mapping = createMapping(filename, testName, defaultTargetPropertyName, userDelta);
         OperationResult opResult = new OperationResult(testName);
-        mapping.evaluate(null, opResult);
+        mapping.evaluate(new NullTaskImpl(), opResult);
         assertResult(opResult);
         PrismValueDeltaSetTriple<PrismPropertyValue<T>> outputTriple = mapping.getOutputTriple();
         if (outputTriple != null) {
@@ -390,7 +391,7 @@ public <T,I> PrismValueDeltaSetTriple<PrismPropertyValue<T>> evaluateMappingDyna
                         valuesToReplace);
         MappingImpl<PrismPropertyValue<T>,PrismPropertyDefinition<T>> mapping = createMapping(filename, testName, defaultTargetPropertyName, userDelta);
         OperationResult opResult = new OperationResult(testName);
-        mapping.evaluate(null, opResult);
+        mapping.evaluate(new NullTaskImpl(), opResult);
         assertResult(opResult);
         PrismValueDeltaSetTriple<PrismPropertyValue<T>> outputTriple = mapping.getOutputTriple();
         if (outputTriple != null) {
@@ -408,7 +409,7 @@ public <T,I> PrismValueDeltaSetTriple<PrismPropertyValue<T>> evaluateMappingDyna
         MappingImpl<PrismPropertyValue<T>,PrismPropertyDefinition<T>> mapping = createMapping(filename, testName, defaultTargetPropertyName, userDelta);
         OperationResult opResult = new OperationResult(testName);
 
-        mapping.evaluate(null, opResult);
+        mapping.evaluate(new NullTaskImpl(), opResult);
         assertResult(opResult);
         PrismValueDeltaSetTriple<PrismPropertyValue<T>> outputTriple = mapping.getOutputTriple();
         if (outputTriple != null) {
@@ -425,7 +426,7 @@ public <T,I> PrismValueDeltaSetTriple<PrismPropertyValue<T>> evaluateMappingDyna
                         valuesToReplace);
         MappingImpl<PrismPropertyValue<T>,PrismPropertyDefinition<T>> mapping = createMapping(filename, testName, defaultTargetPropertyName, userDelta);
         OperationResult opResult = new OperationResult(testName);
-        mapping.evaluate(null, opResult);
+        mapping.evaluate(new NullTaskImpl(), opResult);
         assertResult(opResult);
         PrismValueDeltaSetTriple<PrismPropertyValue<T>> outputTriple = mapping.getOutputTriple();
         if (outputTriple != null) {

model/model-common/src/test/java/com/evolveum/midpoint/model/common/mapping/TestMappingComplex.java

@@ -60,7 +60,7 @@ public void testModifyObjectSetAdditionalName() throws Exception {
         OperationResult opResult = new OperationResult(TEST_NAME);
 
         // WHEN
-        mapping.evaluate(null, opResult);
+        mapping.evaluate(createTask(), opResult);
 
         // THEN
         PrismValueDeltaSetTriple<PrismPropertyValue<PolyString>> outputTriple = mapping.getOutputTriple();
@@ -91,7 +91,7 @@ public void testModifyObjectSetAdditionalNameFalse() throws Exception {
         OperationResult opResult = new OperationResult(TEST_NAME);
 
         // WHEN
-        mapping.evaluate(null, opResult);
+        mapping.evaluate(createTask(), opResult);
 
         // THEN
         PrismValueDeltaSetTriple<PrismPropertyValue<PolyString>> outputTriple = mapping.getOutputTriple();
@@ -118,7 +118,7 @@ public void testModifyObjectUnrelated() throws Exception {
         OperationResult opResult = new OperationResult(TEST_NAME);
 
         // WHEN
-        mapping.evaluate(null, opResult);
+        mapping.evaluate(createTask(), opResult);
 
         // THEN
         PrismValueDeltaSetTriple<PrismPropertyValue<PolyString>> outputTriple = mapping.getOutputTriple();
@@ -148,7 +148,7 @@ public void testModifyObjectUnrelatedFalse() throws Exception {
         OperationResult opResult = new OperationResult(TEST_NAME);
 
         // WHEN
-        mapping.evaluate(null, opResult);
+        mapping.evaluate(createTask(), opResult);
 
         // THEN
         PrismValueDeltaSetTriple<PrismPropertyValue<PolyString>> outputTriple = mapping.getOutputTriple();
@@ -173,7 +173,7 @@ public void testAddObjectUnrelatedFalse() throws Exception {
         OperationResult opResult = new OperationResult(TEST_NAME);
 
         // WHEN
-        mapping.evaluate(null, opResult);
+        mapping.evaluate(createTask(), opResult);
 
         // THEN
         PrismValueDeltaSetTriple<PrismPropertyValue<PolyString>> outputTriple = mapping.getOutputTriple();
@@ -197,7 +197,7 @@ public void testAddObjectUnrelatedEmptyFalse() throws Exception {
         OperationResult opResult = new OperationResult(TEST_NAME);
 
         // WHEN
-        mapping.evaluate(null, opResult);
+        mapping.evaluate(createTask(), opResult);
 
         // THEN
         PrismValueDeltaSetTriple<PrismPropertyValue<PolyString>> outputTriple = mapping.getOutputTriple();

model/model-common/src/test/java/com/evolveum/midpoint/model/common/mapping/TestMappingDomain.java

@@ -11,6 +11,8 @@
 import java.io.IOException;
 import java.util.List;
 
+import com.evolveum.midpoint.task.api.Task;
+import com.evolveum.midpoint.task.api.test.NullTaskImpl;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
 import org.xml.sax.SAXException;
@@ -73,7 +75,7 @@ public void testControlReplaceSingleValue() throws Exception {
         OperationResult opResult = new OperationResult(TEST_NAME);
 
         // WHEN
-        mapping.evaluate(null, opResult);
+        mapping.evaluate(createTask(), opResult);
 
         // THEN
         PrismValueDeltaSetTriple<PrismPropertyValue<PolyString>> outputTriple = mapping.getOutputTriple();
@@ -83,6 +85,10 @@ public void testControlReplaceSingleValue() throws Exception {
           PrismAsserts.assertTripleMinus(outputTriple, PrismTestUtil.createPolyString("Pirate null (1234567890)"));
     }
 
+    private Task createTask() {
+        return new NullTaskImpl();
+    }
+
     /**
      * Control. All goes well here. All values in the domain.
      */
@@ -112,7 +118,7 @@ public void testControlReplaceMultiValue() throws Exception {
         OperationResult opResult = new OperationResult(TEST_NAME);
 
         // WHEN
-        mapping.evaluate(null, opResult);
+        mapping.evaluate(createTask(), opResult);
 
         // THEN
         PrismValueDeltaSetTriple<PrismPropertyValue<PolyString>> outputTriple = mapping.getOutputTriple();
@@ -155,7 +161,7 @@ public void testReplaceMixedMultiValue() throws Exception {
 
         // WHEN
         TestUtil.displayWhen(TEST_NAME);
-        mapping.evaluate(null, opResult);
+        mapping.evaluate(createTask(), opResult);
 
         // THEN
         TestUtil.displayThen(TEST_NAME);
@@ -199,7 +205,7 @@ public void testAddMixedMultiValue() throws Exception {
 
         // WHEN
         TestUtil.displayWhen(TEST_NAME);
-        mapping.evaluate(null, opResult);
+        mapping.evaluate(createTask(), opResult);
 
         // THEN
         TestUtil.displayThen(TEST_NAME);
@@ -246,7 +252,7 @@ public void testDeleteMixedMultiValue() throws Exception {
 
         // WHEN
         TestUtil.displayWhen(TEST_NAME);
-        mapping.evaluate(null, opResult);
+        mapping.evaluate(createTask(), opResult);
 
         // THEN
         TestUtil.displayThen(TEST_NAME);