Improve simulation reports

1. Basic categorization columns (primary archetype, resource, kind,
intent, tag) were added for all simulation reports.

2. "Related assignment" columns were added to item- and value-level
reports. These provide descriptive information regarding assignments
that are added, deleted, or modified.

3. Special "showIfNoDetails" mode parameter was introduced to item/value
simulation reports. It controls whether joins should be left or inner.

Related, potentially dangerous changes:

1. When evaluating scripts with no output definitions, all values were
treated like PPVs. This is not correct, and was fixed in this commit:
Now we wrap values as PPVs/PRVs/PCVs/POVs. This may break some fragile
client code.

2. Pretty-printing code in ColumnDataConverter was improved/fixed,
especially for assignments. They are now displayed in nice form, but
with some information loss compared with original ugly toString dump.
To be reviewed and tested.

Other minor changes:

1. OperationResult handling in ExpressionUtil ref resolution was fixed.

2. "Object after" in ProcessedObject is no longer obtained from
lens context, but from direct delta application on "object before".
It is more precise.

model/model-api/src/main/java/com/evolveum/midpoint/model/api/simulation/ProcessedObject.java

@@ -27,6 +27,7 @@
 import java.io.Serializable;
 import java.util.Collection;
 import java.util.Map;
+import java.util.Set;
 
 /**
  * Parsed analogy of {@link SimulationResultProcessedObjectType}.
@@ -86,11 +87,15 @@ boolean matches(@NotNull SimulationObjectPredicateType predicate, @NotNull Task
 
     interface ProcessedObjectItemDelta<V extends PrismValue, D extends ItemDefinition<?>> extends ItemDelta<V, D> {
         @NotNull Collection<?> getRealValuesBefore();
+        @NotNull Set<? extends PrismValue> getPrismValuesBefore();
         @NotNull Collection<?> getRealValuesAfter();
+        @NotNull Set<? extends PrismValue> getPrismValuesAfter();
         @NotNull Collection<?> getRealValuesAdded();
         @NotNull Collection<?> getRealValuesDeleted();
+        @NotNull Set<?> getRealValuesModified();
         @NotNull Collection<?> getRealValuesUnchanged();
         @NotNull Collection<ValueWithState> getValuesWithStates();
+        @Nullable AssignmentType getRelatedAssignment();
     }
 
     class ValueWithState implements Serializable {
@@ -117,7 +122,7 @@ public String toString() {
         }
 
         public enum State {
-            UNCHANGED, ADDED, DELETED
+            UNCHANGED, ADDED, DELETED, MODIFIED
         }
     }
 

model/model-common/src/main/java/com/evolveum/midpoint/model/common/expression/script/AbstractCachingScriptEvaluator.java

@@ -89,14 +89,14 @@ public <T, V extends PrismValue> List<V> evaluate(ScriptExpressionEvaluationCont
                 if (evalRawResult instanceof Collection) {
                     //noinspection unchecked,rawtypes
                     ((Collection)evalRawResult).forEach(rawResultValue -> {
-                        //noinspection unchecked
-                        evalPrismValues.add(
-                                (V) getPrismContext().itemFactory().createPropertyValue(rawResultValue));
+                        if (rawResultValue != null) {
+                            //noinspection unchecked
+                            evalPrismValues.add((V) toPrismValue(rawResultValue));
+                        }
                     });
                 } else {
                     //noinspection unchecked
-                    evalPrismValues.add(
-                            (V) getPrismContext().itemFactory().createPropertyValue(evalRawResult));
+                    evalPrismValues.add((V) toPrismValue(evalRawResult));
                 }
             }
             return evalPrismValues;
@@ -130,6 +130,19 @@ public <T, V extends PrismValue> List<V> evaluate(ScriptExpressionEvaluationCont
         return values;
     }
 
+    // FIXME this should be some low-level utility method
+    private PrismValue toPrismValue(@NotNull Object realValue) {
+        if (realValue instanceof Objectable) {
+            return ((Objectable) realValue).asPrismObject().getValue();
+        } else if (realValue instanceof Containerable) {
+            return ((Containerable) realValue).asPrismContainerValue();
+        } else if (realValue instanceof Referencable) {
+            return ((Referencable) realValue).asReferenceValue();
+        } else {
+            return getPrismContext().itemFactory().createPropertyValue(realValue);
+        }
+    }
+
     @NotNull
     private <T> Class<T> determineJavaReturnType(ScriptExpressionEvaluationContext context) {
         QName xsdReturnType = context.getOutputDefinition().getTypeName();

model/model-common/src/main/java/com/evolveum/midpoint/model/common/expression/script/AbstractScriptEvaluator.java

@@ -103,15 +103,16 @@ protected Map<String, Object> prepareScriptVariablesValueMap(ScriptExpressionEva
                         context.getExpressionType().getValueVariableMode(), ValueVariableModeType.REAL_VALUE);
 
                 //noinspection rawtypes
-                TypedValue variableTypedValue = ExpressionUtil.convertVariableValue(variableEntry.getValue(), variableName,
+                TypedValue variableTypedValue = ExpressionUtil.convertVariableValue(
+                        variableEntry.getValue(), variableName,
                         context.getObjectResolver(), context.getContextDescription(),
                         context.getExpressionType().getObjectVariableMode(),
                         valueVariableMode,
                         prismContext, context.getTask(), context.getResult());
 
                 scriptVariableMap.put(variableName, variableTypedValue.getValue());
                 if (context.getTrace() != null && !variables.isAlias(variableName)) {
-                    ScriptVariableEvaluationTraceType variableTrace = new ScriptVariableEvaluationTraceType(prismContext);
+                    ScriptVariableEvaluationTraceType variableTrace = new ScriptVariableEvaluationTraceType();
                     variableTrace.setName(new QName(variableName));
                     Object clonedValue = cloneIfPossible(variableTypedValue.getValue());
                     variableTrace.getValue().addAll(TraceUtil.toAnyValueTypeList(clonedValue, prismContext));

model/model-common/src/main/java/com/evolveum/midpoint/model/common/expression/script/ScriptExpression.java

@@ -92,19 +92,11 @@ public void setExpressionProfile(ExpressionProfile expressionProfile) {
         this.expressionProfile = expressionProfile;
     }
 
-    public ScriptExpressionProfile getScriptExpressionProfile() {
-        return scriptExpressionProfile;
-    }
-
-    public void setScriptExpressionProfile(ScriptExpressionProfile scriptExpressionProfile) {
+    void setScriptExpressionProfile(ScriptExpressionProfile scriptExpressionProfile) {
         this.scriptExpressionProfile = scriptExpressionProfile;
     }
 
-    public Function<Object, Object> getAdditionalConvertor() {
-        return additionalConvertor;
-    }
-
-    public void setAdditionalConvertor(Function<Object, Object> additionalConvertor) {
+    void setAdditionalConvertor(Function<Object, Object> additionalConvertor) {
         this.additionalConvertor = additionalConvertor;
     }
 
@@ -148,7 +140,7 @@ public <V extends PrismValue> List<V> evaluate(ScriptExpressionEvaluationContext
                 .addContext("context", context.getContextDescription())
                 .build();
         if (result.isTracingNormal(ScriptEvaluationTraceType.class)) {
-            ScriptEvaluationTraceType trace = new ScriptEvaluationTraceType(prismContext);
+            ScriptEvaluationTraceType trace = new ScriptEvaluationTraceType();
             result.addTrace(trace);
             context.setTrace(trace);
             trace.setScriptExpressionEvaluator(context.getExpressionType());
@@ -210,6 +202,7 @@ private void traceExpressionFailure(ScriptExpressionEvaluationContext context, T
                 formatProfile(), formatCode(), SchemaDebugUtil.prettyPrint(exception));
     }
 
+    @SuppressWarnings("BooleanMethodIsAlwaysInverted")
     private boolean isTrace() {
         return LOGGER.isTraceEnabled() || (scriptType != null && scriptType.isTrace() == Boolean.TRUE);
     }

model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/simulation/ProcessedObjectImpl.java

@@ -9,6 +9,7 @@
 
 import static com.evolveum.midpoint.schema.util.ObjectReferenceTypeUtil.getTargetNameOrOid;
 import static com.evolveum.midpoint.schema.util.ObjectTypeUtil.asObjectable;
+import static com.evolveum.midpoint.schema.util.ObjectTypeUtil.asPrismObject;
 import static com.evolveum.midpoint.schema.util.SimulationMetricPartitionTypeUtil.ALL_DIMENSIONS;
 import static com.evolveum.midpoint.util.MiscUtil.argCheck;
 import static com.evolveum.midpoint.util.MiscUtil.emptyIfNull;
@@ -19,14 +20,6 @@
 import java.util.stream.Collectors;
 import javax.xml.namespace.QName;
 
-import com.evolveum.midpoint.prism.*;
-import com.evolveum.midpoint.prism.deleg.ItemDeltaDelegator;
-import com.evolveum.midpoint.prism.path.ItemPath;
-import com.evolveum.midpoint.provisioning.api.ShadowSimulationData;
-import com.evolveum.midpoint.schema.util.delta.ItemDeltaFilter;
-import com.evolveum.midpoint.util.annotation.Experimental;
-
-import com.google.common.collect.Sets;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 import org.jetbrains.annotations.VisibleForTesting;
@@ -36,9 +29,14 @@
 import com.evolveum.midpoint.model.impl.ModelBeans;
 import com.evolveum.midpoint.model.impl.lens.LensElementContext;
 import com.evolveum.midpoint.model.impl.lens.LensFocusContext;
+import com.evolveum.midpoint.prism.*;
+import com.evolveum.midpoint.prism.deleg.ItemDeltaDelegator;
 import com.evolveum.midpoint.prism.delta.ItemDelta;
 import com.evolveum.midpoint.prism.delta.ObjectDelta;
+import com.evolveum.midpoint.prism.equivalence.ParameterizedEquivalenceStrategy;
+import com.evolveum.midpoint.prism.path.ItemPath;
 import com.evolveum.midpoint.prism.query.ObjectFilter;
+import com.evolveum.midpoint.provisioning.api.ShadowSimulationData;
 import com.evolveum.midpoint.repo.common.expression.ExpressionUtil;
 import com.evolveum.midpoint.schema.DeltaConvertor;
 import com.evolveum.midpoint.schema.SchemaService;
@@ -49,10 +47,12 @@
 import com.evolveum.midpoint.schema.simulation.PartitionScope;
 import com.evolveum.midpoint.schema.util.MiscSchemaUtil;
 import com.evolveum.midpoint.schema.util.ObjectTypeUtil;
+import com.evolveum.midpoint.schema.util.delta.ItemDeltaFilter;
 import com.evolveum.midpoint.task.api.Task;
 import com.evolveum.midpoint.util.DebugDumpable;
 import com.evolveum.midpoint.util.DebugUtil;
 import com.evolveum.midpoint.util.MiscUtil;
+import com.evolveum.midpoint.util.annotation.Experimental;
 import com.evolveum.midpoint.util.exception.CommonException;
 import com.evolveum.midpoint.util.exception.SchemaException;
 import com.evolveum.midpoint.util.exception.SystemException;
@@ -202,12 +202,20 @@ static <O extends ObjectType> ProcessedObjectImpl<O> createSingleDelta(
 
         Class<O> type = elementContext.getObjectTypeClass();
         @Nullable O stateBefore = asObjectable(elementContext.getStateBeforeSimulatedOperation());
-        @Nullable O stateAfter = asObjectable(elementContext.getObjectNew());
-        @Nullable O anyState = MiscUtil.getFirstNonNull(stateAfter, stateBefore);
         @Nullable ObjectDelta<O> delta = singleDelta != null ? singleDelta : elementContext.getSummaryExecutedDelta();
-        if (anyState == null || stateBefore == null && delta == null) {
-            // Nothing to report on here. Note that for shadows it's possible that stateBefore and delta are null but
-            // stateAfter is "PCV(null)" - due to the way of computation of objectNew for shadows
+
+        // We intentionally do not use "object new" because it does not contain e.g. linkRefs added.
+        @Nullable O stateAfter;
+        if (delta == null) {
+            stateAfter = stateBefore;
+        } else {
+            stateAfter = asObjectable(
+                    delta.computeChangedObject(
+                            asPrismObject(stateBefore)));
+        }
+        @Nullable O anyState = MiscUtil.getFirstNonNull(stateAfter, stateBefore);
+
+        if (anyState == null) {
             return null;
         }
 
@@ -782,10 +790,18 @@ private Set<?> getRealValuesBefore(@NotNull ItemPath path) {
         return getRealValues(before, path);
     }
 
+    private Set<? extends PrismValue> getPrismValuesBefore(@NotNull ItemPath path) {
+        return getPrismValues(before, path);
+    }
+
     private Set<?> getRealValuesAfter(@NotNull ItemPath path) {
         return getRealValues(after, path);
     }
 
+    private Set<? extends PrismValue> getPrismValuesAfter(@NotNull ItemPath path) {
+        return getPrismValues(after, path);
+    }
+
     @Override
     public @NotNull Collection<ProcessedObjectItemDelta<?, ?>> getItemDeltas(
             @Nullable Object pathsToInclude, @Nullable Object pathsToExclude, @Nullable Boolean includeOperationalItems) {
@@ -801,12 +817,18 @@ private Set<?> getRealValuesAfter(@NotNull ItemPath path) {
     }
 
     private Set<?> getRealValues(O object, ItemPath path) {
+        return getPrismValues(object, path).stream()
+                .map(v -> v.getRealValue())
+                .collect(Collectors.toSet());
+    }
+
+    private Set<? extends PrismValue> getPrismValues(O object, ItemPath path) {
         if (object == null) {
             return Set.of();
         }
         Item<?, ?> item = object.asPrismContainerValue().findItem(path);
         return item != null ?
-                Set.copyOf(item.getRealValues()) :
+                Set.copyOf(item.getValues()) :
                 Set.of();
     }
 
@@ -835,35 +857,88 @@ public ItemDelta<V, D> clone() {
             return ProcessedObjectImpl.this.getRealValuesBefore(getPath());
         }
 
+        @Override
+        public @NotNull Set<? extends PrismValue> getPrismValuesBefore() {
+            return ProcessedObjectImpl.this.getPrismValuesBefore(getPath());
+        }
+
         @Override
         public @NotNull Set<?> getRealValuesAfter() {
             return ProcessedObjectImpl.this.getRealValuesAfter(getPath());
         }
 
+        @Override
+        public @NotNull Set<? extends PrismValue> getPrismValuesAfter() {
+            return ProcessedObjectImpl.this.getPrismValuesAfter(getPath());
+        }
+
         @Override
         public @NotNull Set<?> getRealValuesAdded() {
-            return Sets.difference(getRealValuesAfter(), getRealValuesBefore());
+            return PrismValueCollectionsUtil.getRealValuesOfCollection(
+                    PrismValueCollectionsUtil.differenceConsideringIds(
+                            getPrismValuesAfter(),
+                            getPrismValuesBefore(),
+                            ParameterizedEquivalenceStrategy.REAL_VALUE));
         }
 
         @Override
         public @NotNull Set<?> getRealValuesDeleted() {
-            return Sets.difference(getRealValuesBefore(), getRealValuesAfter());
+            return PrismValueCollectionsUtil.getRealValuesOfCollection(
+                    PrismValueCollectionsUtil.differenceConsideringIds(
+                            getPrismValuesBefore(),
+                            getPrismValuesAfter(),
+                            ParameterizedEquivalenceStrategy.REAL_VALUE));
+        }
+
+        @Override
+        public @NotNull Set<?> getRealValuesModified() {
+            return PrismValueCollectionsUtil.getRealValuesOfCollection(
+                    PrismValueCollectionsUtil.sameIdDifferentContent(
+                            getPrismValuesBefore(),
+                            getPrismValuesAfter(),
+                            ParameterizedEquivalenceStrategy.REAL_VALUE));
         }
 
         @Override
         public @NotNull Set<?> getRealValuesUnchanged() {
-            return Sets.intersection(getRealValuesBefore(), getRealValuesAfter());
+            return PrismValueCollectionsUtil.getRealValuesOfCollection(
+                    PrismValueCollectionsUtil.intersection(
+                            getPrismValuesBefore(),
+                            getPrismValuesAfter(),
+                            ParameterizedEquivalenceStrategy.REAL_VALUE));
         }
 
         @Override
         public @NotNull Collection<ValueWithState> getValuesWithStates() {
             List<ValueWithState> all = new ArrayList<>();
             getRealValuesAdded().forEach(v -> all.add(new ValueWithState(v, ValueWithState.State.ADDED)));
             getRealValuesDeleted().forEach(v -> all.add(new ValueWithState(v, ValueWithState.State.DELETED)));
+            getRealValuesModified().forEach(v -> all.add(new ValueWithState(v, ValueWithState.State.MODIFIED)));
             getRealValuesUnchanged().forEach(v -> all.add(new ValueWithState(v, ValueWithState.State.UNCHANGED)));
             return all;
         }
 
+        @Override
+        public @Nullable AssignmentType getRelatedAssignment() {
+            ItemPath path = getPath();
+            if (!path.startsWith(AssignmentHolderType.F_ASSIGNMENT)) {
+                return null;
+            }
+            ItemPath rest = path.rest();
+            Long id = rest.firstToIdOrNull();
+            if (id == null) {
+                return null;
+            }
+            O any = MiscUtil.getFirstNonNull(after, before);
+            if (!(any instanceof AssignmentHolderType)) {
+                return null; // should not occur
+            }
+            return ((AssignmentHolderType) any).getAssignment().stream()
+                    .filter(a -> id.equals(a.getId()))
+                    .findFirst()
+                    .orElse(null);
+        }
+
         @Override
         public String toString() {
             return "ProcessedObjectItemDelta{" + delegate + '}';