Compute simulation metrics incrementally

This commit switches computation of simulation metrics from "at the end
of simulation" (time-consuming, late, failing on suspend/resume) to
"at the end of each bucket".

To support this, a new concept of "simulation result transaction" was
introduced.

Other improvements:

1. LensElementContext is now available for custom metrics computation.
2. Metric values summation and (partial/total) dimension collapsing
are now implemented correctly.

Work in progress. E.g. simulation result transactions still have to be
implemented in the repository.

infra/schema/src/main/java/com/evolveum/midpoint/schema/TaskExecutionMode.java

@@ -47,6 +47,10 @@ public boolean isPersistent() {
         return persistent;
     }
 
+    public boolean isSimulation() {
+        return !persistent;
+    }
+
     /**
      * What configuration should the actions take into account? Production or "development" one?
      *

infra/schema/src/main/java/com/evolveum/midpoint/schema/constants/ExpressionConstants.java

@@ -45,6 +45,7 @@ public class ExpressionConstants {
     public static final String VAR_RESOURCE = "resource";
     public static final String VAR_DELTA = "delta";
     public static final String VAR_MODEL_CONTEXT = "modelContext";
+    public static final String VAR_MODEL_ELEMENT_CONTEXT = "modelElementContext";
     public static final String VAR_PRISM_CONTEXT = "prismContext";
     public static final String VAR_LOCALIZATION_SERVICE = "localizationService";
     public static final String VAR_CONFIGURATION = "configuration";

infra/schema/src/main/java/com/evolveum/midpoint/schema/constants/SchemaConstants.java

@@ -665,4 +665,6 @@ public abstract class SchemaConstants {
             ItemPath.create(FocusType.F_IDENTITIES, FocusIdentitiesType.F_NORMALIZED_DATA);
     public static final @NotNull ItemPath PATH_FOCUS_DEFAULT_AUTHORITATIVE_SOURCE =
             ItemPath.create(FocusType.F_IDENTITIES, FocusIdentitiesType.F_DEFAULT_AUTHORITATIVE_SOURCE);
+
+    public static final String SIMULATION_RESULT_DEFAULT_TRANSACTION_ID = "default";
 }

infra/schema/src/main/java/com/evolveum/midpoint/schema/simulation/PartitionScope.java

@@ -7,12 +7,20 @@
 
 package com.evolveum.midpoint.schema.simulation;
 
+import com.evolveum.midpoint.prism.path.ItemName;
+import com.evolveum.midpoint.schema.util.SimulationMetricPartitionScopeTypeUtil;
 import com.evolveum.midpoint.xml.ns._public.common.common_3.ShadowKindType;
 import com.evolveum.midpoint.xml.ns._public.common.common_3.SimulationMetricPartitionScopeType;
 
+import com.google.common.collect.Sets;
+
 import javax.xml.namespace.QName;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Objects;
+import java.util.Set;
+
+import static com.evolveum.midpoint.xml.ns._public.common.common_3.SimulationMetricPartitionScopeType.*;
 
 /**
  * Parsed form of {@link SimulationMetricPartitionScopeType}.
@@ -25,23 +33,36 @@ public class PartitionScope {
     private final String resourceOid;
     private final ShadowKindType kind;
     private final String intent;
+    private final Set<QName> allDimensions;
 
-    public PartitionScope(QName objectType, String resourceOid, ShadowKindType kind, String intent) {
+    public PartitionScope(QName objectType, String resourceOid, ShadowKindType kind, String intent, Set<QName> allDimensions) {
         this.objectType = objectType;
         this.resourceOid = resourceOid;
         this.kind = kind;
         this.intent = intent;
+        this.allDimensions = new HashSet<>(allDimensions);
     }
 
-    public static PartitionScope fromBean(SimulationMetricPartitionScopeType scope) {
+    public static PartitionScope fromBean(SimulationMetricPartitionScopeType scope, Set<QName> availableDimensions) {
         if (scope == null) {
-            return new PartitionScope(null, null, null, null);
+            return new PartitionScope(null, null, null, null, availableDimensions);
         } else {
             return new PartitionScope(
-                    scope.getTypeName(),
-                    scope.getResourceOid(),
-                    scope.getKind(),
-                    scope.getIntent());
+                    ifAvailable(scope.getTypeName(), availableDimensions, F_TYPE_NAME),
+                    ifAvailable(scope.getResourceOid(), availableDimensions, F_RESOURCE_OID),
+                    ifAvailable(scope.getKind(), availableDimensions, F_KIND),
+                    ifAvailable(scope.getIntent(), availableDimensions, F_INTENT),
+                    Sets.intersection(
+                            SimulationMetricPartitionScopeTypeUtil.getDimensions(scope),
+                            availableDimensions));
+        }
+    }
+
+    private static <T> T ifAvailable(T value, Set<QName> availableDimensions, ItemName dimensionName) {
+        if (value != null && availableDimensions.contains(dimensionName)) {
+            return value;
+        } else {
+            return null;
         }
     }
 
@@ -57,7 +78,8 @@ public boolean equals(Object o) {
         return Objects.equals(objectType, that.objectType)
                 && Objects.equals(resourceOid, that.resourceOid)
                 && kind == that.kind
-                && Objects.equals(intent, that.intent);
+                && Objects.equals(intent, that.intent)
+                && Objects.equals(allDimensions, that.allDimensions);
     }
 
     @Override
@@ -66,24 +88,23 @@ public int hashCode() {
     }
 
     public SimulationMetricPartitionScopeType toBean() {
-        // FIXME fill nullDimensions correctly
         SimulationMetricPartitionScopeType bean = new SimulationMetricPartitionScopeType()
                 .typeName(objectType)
                 .resourceOid(resourceOid)
                 .kind(kind)
                 .intent(intent);
         List<QName> nullDimensions = bean.getNullDimensions();
-        if (objectType == null) {
-            nullDimensions.add(SimulationMetricPartitionScopeType.F_TYPE_NAME);
+        if (objectType == null && allDimensions.contains(F_TYPE_NAME)) {
+            nullDimensions.add(F_TYPE_NAME);
         }
-        if (resourceOid == null) {
-            nullDimensions.add(SimulationMetricPartitionScopeType.F_RESOURCE_OID);
+        if (resourceOid == null && allDimensions.contains(F_RESOURCE_OID)) {
+            nullDimensions.add(F_RESOURCE_OID);
         }
-        if (kind == null) {
-            nullDimensions.add(SimulationMetricPartitionScopeType.F_KIND);
+        if (kind == null && allDimensions.contains(F_KIND)) {
+            nullDimensions.add(F_KIND);
         }
-        if (intent == null) {
-            nullDimensions.add(SimulationMetricPartitionScopeType.F_INTENT);
+        if (intent == null && allDimensions.contains(F_INTENT)) {
+            nullDimensions.add(F_INTENT);
         }
         return bean;
     }

infra/schema/src/main/java/com/evolveum/midpoint/schema/simulation/SimulationMetricComputer.java

@@ -7,15 +7,20 @@
 
 package com.evolveum.midpoint.schema.simulation;
 
-import static com.evolveum.midpoint.schema.util.SimulationMetricValuesTypeUtil.selectPartitions;
+import static com.evolveum.midpoint.schema.util.SimulationMetricPartitionTypeCollectionUtil.selectPartitions;
+import static com.evolveum.midpoint.schema.util.SimulationMetricPartitionTypeUtil.ALL_DIMENSIONS;
 
 import java.math.BigDecimal;
 import java.math.RoundingMode;
+import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import javax.xml.namespace.QName;
 
 import com.evolveum.midpoint.schema.util.SimulationMetricPartitionDimensionsTypeUtil;
+import com.evolveum.midpoint.schema.util.SimulationMetricValuesTypeCollectionUtil;
+import com.evolveum.midpoint.xml.ns._public.common.common_3.SimulationMetricReferenceType;
 import com.evolveum.midpoint.xml.ns._public.common.common_3.SimulationMetricValuesType;
 
 import com.google.common.collect.Sets;
@@ -24,6 +29,8 @@
 import com.evolveum.midpoint.xml.ns._public.common.common_3.SimulationMetricAggregationFunctionType;
 import com.evolveum.midpoint.xml.ns._public.common.common_3.SimulationMetricPartitionType;
 
+import org.jetbrains.annotations.Nullable;
+
 /**
  * Works with the metric computations at one place.
  *
@@ -33,6 +40,52 @@ public class SimulationMetricComputer {
 
     private static final int DEFAULT_SCALE = 10;
 
+    /**
+     * Computes "base + delta", according to all dimensions that are present.
+     * We assume these dimensions are "compatible". (TODO)
+     */
+    public static @NotNull List<SimulationMetricValuesType> add(
+            @NotNull List<SimulationMetricValuesType> base,
+            @NotNull List<SimulationMetricValuesType> delta) {
+        List<SimulationMetricValuesType> sum = new ArrayList<>();
+        Set<SimulationMetricReferenceType> presentInDelta = new HashSet<>();
+        for (SimulationMetricValuesType deltaMetric : delta) {
+            SimulationMetricReferenceType deltaRef = deltaMetric.getRef();
+            presentInDelta.add(deltaRef);
+            SimulationMetricValuesType matchingInBase =
+                    SimulationMetricValuesTypeCollectionUtil.findByRef(base, deltaRef);
+            sum.add(
+                    add(matchingInBase, deltaMetric));
+        }
+        for (SimulationMetricValuesType baseMetric : base) {
+            if (!presentInDelta.contains(baseMetric.getRef())) {
+                sum.add(baseMetric.clone());
+            }
+        }
+        return sum;
+    }
+
+    private static @NotNull SimulationMetricValuesType add(
+            @Nullable SimulationMetricValuesType base,
+            @NotNull SimulationMetricValuesType delta) {
+        if (base == null) {
+            return delta.clone();
+        }
+        SimulationMetricPartitions sumPartitions = new SimulationMetricPartitions(ALL_DIMENSIONS);
+        for (SimulationMetricPartitionType basePartition : base.getPartition()) {
+            sumPartitions.addPartition(basePartition);
+        }
+        for (SimulationMetricPartitionType deltaPartition : delta.getPartition()) {
+            sumPartitions.addPartition(deltaPartition);
+        }
+        // TODO we could check if aggregation function and source dimensions match
+        SimulationMetricValuesType sum = base.clone();
+        sum.getPartition().clear();
+        sum.getPartition().addAll(
+                sumPartitions.toPartitionBeans(delta.getAggregationFunction()));
+        return sum;
+    }
+
     public static List<SimulationMetricPartitionType> computePartitions(
             @NotNull SimulationMetricValuesType mv, @NotNull Set<QName> dimensions) {
         Set<QName> sourceDimensions = SimulationMetricPartitionDimensionsTypeUtil.getDimensions(mv.getSourceDimensions());
@@ -43,8 +96,8 @@ public static List<SimulationMetricPartitionType> computePartitions(
                             "Cannot compute partition for %s as the following dimension(s) are missing: %s; source = %s",
                             dimensions, missing, sourceDimensions));
         }
-        SimulationMetricPartitions targetPartitions = new SimulationMetricPartitions();
-        for (SimulationMetricPartitionType sourcePartitionBean : selectPartitions(mv, sourceDimensions)) {
+        SimulationMetricPartitions targetPartitions = new SimulationMetricPartitions(dimensions);
+        for (SimulationMetricPartitionType sourcePartitionBean : selectPartitions(mv.getPartition(), sourceDimensions)) {
             targetPartitions.addPartition(sourcePartitionBean);
         }
         return targetPartitions.toPartitionBeans(mv.getAggregationFunction());

infra/schema/src/main/java/com/evolveum/midpoint/schema/simulation/SimulationMetricPartition.java

@@ -17,7 +17,7 @@
 /**
  * Parsed form of {@link SimulationMetricPartitionType}.
  *
- * Created for fast and simple aggregation.
+ * Created for fast, simple, and thread-safe aggregation.
  */
 public class SimulationMetricPartition {
 
@@ -32,7 +32,7 @@ public class SimulationMetricPartition {
     private BigDecimal domainMaxValue;
 
     @SuppressWarnings("DuplicatedCode")
-    public void addObject(BigDecimal sourceMetricValue, boolean inSelection) {
+    public synchronized void addObject(BigDecimal sourceMetricValue, boolean inSelection) {
         if (domainMinValue == null || sourceMetricValue.compareTo(domainMinValue) < 0) {
             domainMinValue = sourceMetricValue;
         }
@@ -54,7 +54,7 @@ public void addObject(BigDecimal sourceMetricValue, boolean inSelection) {
         }
     }
 
-    void addPartition(SimulationMetricPartitionType other) {
+    synchronized void addOtherPartition(SimulationMetricPartitionType other) {
         BigDecimal otherDomainMinValue = other.getDomainMinValue();
         if (domainMinValue == null || otherDomainMinValue != null && otherDomainMinValue.compareTo(domainMinValue) < 0) {
             domainMinValue = otherDomainMinValue;
@@ -78,7 +78,8 @@ void addPartition(SimulationMetricPartitionType other) {
         selectionTotalValue = selectionTotalValue.add(or0(other.getSelectionTotalValue()));
     }
 
-    public SimulationMetricPartitionType toBean(PartitionScope key, SimulationMetricAggregationFunctionType function) {
+    public synchronized SimulationMetricPartitionType toBean(
+            PartitionScope key, SimulationMetricAggregationFunctionType function) {
         var bean = toBean(key);
         bean.setValue(
                 SimulationMetricComputer.computeValue(bean, function, null));

infra/schema/src/main/java/com/evolveum/midpoint/schema/simulation/SimulationMetricPartitions.java

@@ -8,19 +8,29 @@
 package com.evolveum.midpoint.schema.simulation;
 
 import java.math.BigDecimal;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.stream.Collectors;
 
 import org.jetbrains.annotations.NotNull;
 
 import com.evolveum.midpoint.xml.ns._public.common.common_3.SimulationMetricAggregationFunctionType;
 import com.evolveum.midpoint.xml.ns._public.common.common_3.SimulationMetricPartitionType;
 
+import javax.xml.namespace.QName;
+
 public class SimulationMetricPartitions {
 
-    @NotNull private final Map<PartitionScope, SimulationMetricPartition> partitions = new HashMap<>();
+    /** Dimensions according to which we want to aggregate the partitions. */
+    @NotNull private final Set<QName> dimensions;
+
+    @NotNull private final Map<PartitionScope, SimulationMetricPartition> partitions = new ConcurrentHashMap<>();
+
+    public SimulationMetricPartitions(@NotNull Set<QName> dimensions) {
+        this.dimensions = dimensions;
+    }
 
     public List<SimulationMetricPartitionType> toPartitionBeans(@NotNull SimulationMetricAggregationFunctionType function) {
         return partitions.entrySet().stream()
@@ -35,9 +45,9 @@ public void addObject(PartitionScope key, BigDecimal sourceMetricValue, boolean
     }
 
     void addPartition(SimulationMetricPartitionType sourcePartitionBean) {
-        PartitionScope key = PartitionScope.fromBean(sourcePartitionBean.getScope());
+        PartitionScope key = PartitionScope.fromBean(sourcePartitionBean.getScope(), dimensions);
         partitions
                 .computeIfAbsent(key, (k) -> new SimulationMetricPartition())
-                .addPartition(sourcePartitionBean);
+                .addOtherPartition(sourcePartitionBean);
     }
 }

infra/schema/src/main/java/com/evolveum/midpoint/schema/util/SimulationMetricPartitionTypeCollectionUtil.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright (C) 2010-2023 Evolveum and contributors
+ *
+ * This work is dual-licensed under the Apache License 2.0
+ * and European Union Public License. See LICENSE file for details.
+ */
+
+package com.evolveum.midpoint.schema.util;
+
+import com.evolveum.midpoint.xml.ns._public.common.common_3.SimulationMetricPartitionType;
+
+import org.jetbrains.annotations.NotNull;
+
+import javax.xml.namespace.QName;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public class SimulationMetricPartitionTypeCollectionUtil {
+
+    public static List<SimulationMetricPartitionType> selectPartitions(
+            @NotNull List<SimulationMetricPartitionType> allPartitions, @NotNull Set<QName> dimensions) {
+        return allPartitions.stream()
+                .filter(p -> SimulationMetricPartitionTypeUtil.matches(p, dimensions))
+                .collect(Collectors.toList());
+    }
+}

infra/schema/src/main/java/com/evolveum/midpoint/schema/util/SimulationMetricValuesTypeCollectionUtil.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright (C) 2010-2023 Evolveum and contributors
+ *
+ * This work is dual-licensed under the Apache License 2.0
+ * and European Union Public License. See LICENSE file for details.
+ */
+
+package com.evolveum.midpoint.schema.util;
+
+import com.evolveum.midpoint.util.MiscUtil;
+import com.evolveum.midpoint.xml.ns._public.common.common_3.SimulationMetricReferenceType;
+import com.evolveum.midpoint.xml.ns._public.common.common_3.SimulationMetricValuesType;
+
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.Collection;
+import java.util.stream.Collectors;
+
+public class SimulationMetricValuesTypeCollectionUtil {
+
+    public @Nullable static SimulationMetricValuesType findByRef(
+            @NotNull Collection<SimulationMetricValuesType> metrics,
+            @NotNull SimulationMetricReferenceType ref) {
+        var matching = metrics.stream()
+                .filter(mv -> ref.equals(mv.getRef()))
+                .collect(Collectors.toList());
+        return MiscUtil.extractSingleton(
+                matching,
+                () -> new IllegalStateException(
+                        String.format("Multiple occurrences of '%s': %s", ref, matching)));
+    }
+}