Add skeleton of cluster-wide thresholds

Here we introduce cluster-wide, persistent, activity-level counters.
They can be used for various things, but here are used to count
how many times individual policy rules have been triggered.

Other changes:

1) Worker task now carries information about current activity execution
(ExecutionContext interface). It is to access the execution mode
and to update the counters stored in activity state.

2) Introduced separate projector step: policy rules counters (after
focus iteration) that updates counters for thresholded policy rules.

3) Moved SuspendTaskExecutor execution to the first clockwork run,
just after Enforcer execution. They could be eventually merged into one
component.

4) Fixed policy rule id generation. Now it contains defining object
OID plus rule or assignment ID (for global/assigned policy rules).

5) Other minor fixes/improvements, like pulling everything related to
policy rules into separate PolicyRulesContext in LensElementContext.

Work in progress. Only live sync thresholds tests pass. Recon tests
are not migrated. Multi-node scenarios are not fully implemented.

gui/admin-gui/src/main/java/com/evolveum/midpoint/gui/api/page/PageBase.java

@@ -16,8 +16,6 @@
 
 import com.evolveum.midpoint.gui.impl.component.menu.LeftMenuPanel;
 
-import com.evolveum.midpoint.report.api.ReportService;
-
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.Validate;
@@ -91,7 +89,7 @@
 import com.evolveum.midpoint.prism.polystring.PolyString;
 import com.evolveum.midpoint.prism.query.QueryConverter;
 import com.evolveum.midpoint.repo.api.CacheDispatcher;
-import com.evolveum.midpoint.repo.api.CounterManager;
+import com.evolveum.midpoint.repo.common.activity.CounterManager;
 import com.evolveum.midpoint.repo.api.RepositoryService;
 import com.evolveum.midpoint.repo.common.ObjectResolver;
 import com.evolveum.midpoint.repo.common.expression.Expression;

gui/admin-gui/src/main/java/com/evolveum/midpoint/web/page/admin/configuration/InternalsCountersPanel.java

@@ -28,7 +28,7 @@
 import org.apache.wicket.model.IModel;
 
 import com.evolveum.midpoint.gui.api.component.BasePanel;
-import com.evolveum.midpoint.repo.api.CounterSpecification;
+import com.evolveum.midpoint.repo.common.activity.CounterSpecification;
 import com.evolveum.midpoint.schema.internals.InternalCounters;
 import com.evolveum.midpoint.schema.internals.InternalMonitor;
 import com.evolveum.midpoint.web.component.dialog.ConfirmationPanel;

infra/schema/src/main/java/com/evolveum/midpoint/schema/util/PolicyRuleTypeUtil.java

@@ -645,4 +645,12 @@ private static void getExclusionTriggersFromTriggers(List<EvaluatedExclusionTrig
             }
         }
     }
+
+    public static String createId(String containingObjectOid, Long containerId) {
+        return containingObjectOid + ":" + containerId;
+    }
+
+    public static String createId(String containingObjectOid) {
+        return containingObjectOid + ":rule";
+    }
 }

infra/schema/src/main/resources/xml/ns/public/common/common-core-3.xsd

@@ -19722,6 +19722,7 @@
             <xsd:element name="objectTemplateAfterAssignments" type="tns:PartialProcessingTypeType" minOccurs="0" default="automatic"/>
             <xsd:element name="focusCredentials" type="tns:PartialProcessingTypeType" minOccurs="0" default="automatic"/>
             <xsd:element name="focusPolicyRules" type="tns:PartialProcessingTypeType" minOccurs="0" default="automatic"/>
+            <xsd:element name="policyRuleCounters" type="tns:PartialProcessingTypeType" minOccurs="0" default="automatic"/>
             <xsd:element name="projection" type="tns:PartialProcessingTypeType" minOccurs="0" default="automatic"/>
             <xsd:element name="outbound" type="tns:PartialProcessingTypeType" minOccurs="0" default="automatic"/>
             <xsd:element name="projectionValues" type="tns:PartialProcessingTypeType" minOccurs="0" default="automatic"/>

infra/schema/src/main/resources/xml/ns/public/common/common-tasks-3.xsd

@@ -2713,6 +2713,16 @@
                     </xsd:documentation>
                 </xsd:annotation>
             </xsd:element>
+            <xsd:element name="counters" type="tns:ActivityCounterGroupsType" minOccurs="0">
+                <xsd:annotation>
+                    <xsd:documentation>
+                        Counters related to the execution of the activity, grouped into related categories.
+                        One of the categories are counters needed to implement thresholds for policy rules.
+
+                        Note: The counters may be moved to separate repository object later to improve performance.
+                    </xsd:documentation>
+                </xsd:annotation>
+            </xsd:element>
             <xsd:element name="workState" type="tns:AbstractActivityWorkStateType" minOccurs="0">
                 <xsd:annotation>
                     <xsd:documentation>
@@ -6857,4 +6867,81 @@
             </xsd:element>
         </xsd:sequence>
     </xsd:complexType>
+
+    <xsd:complexType name="ActivityCounterGroupsType">
+        <xsd:annotation>
+            <xsd:documentation>
+                Counters related to the execution of the activity, grouped into related categories.
+            </xsd:documentation>
+            <xsd:appinfo>
+                <a:container>true</a:container>
+                <a:since>4.4</a:since>
+                <a:experimental>true</a:experimental>
+            </xsd:appinfo>
+        </xsd:annotation>
+        <xsd:sequence>
+            <xsd:element name="policyRules" type="tns:ActivityCounterGroupType" minOccurs="0">
+                <xsd:annotation>
+                    <xsd:documentation>
+                        Counters for policy rules.
+                    </xsd:documentation>
+                </xsd:annotation>
+            </xsd:element>
+        </xsd:sequence>
+        <xsd:attribute name="id" type="xsd:long"/>
+    </xsd:complexType>
+
+    <xsd:complexType name="ActivityCounterGroupType">
+        <xsd:annotation>
+            <xsd:documentation>
+                A group of related counters.
+            </xsd:documentation>
+            <xsd:appinfo>
+                <a:container>true</a:container>
+                <a:since>4.4</a:since>
+                <a:experimental>true</a:experimental>
+            </xsd:appinfo>
+        </xsd:annotation>
+        <xsd:sequence>
+            <xsd:element name="counter" type="tns:ActivityCounterType" minOccurs="0" maxOccurs="unbounded">
+                <xsd:annotation>
+                    <xsd:documentation>
+                        A single counter.
+                    </xsd:documentation>
+                </xsd:annotation>
+            </xsd:element>
+        </xsd:sequence>
+        <xsd:attribute name="id" type="xsd:long"/>
+    </xsd:complexType>
+
+    <xsd:complexType name="ActivityCounterType">
+        <xsd:annotation>
+            <xsd:documentation>
+                A single counter.
+            </xsd:documentation>
+            <xsd:appinfo>
+                <a:container>true</a:container>
+                <a:since>4.4</a:since>
+                <a:experimental>true</a:experimental>
+            </xsd:appinfo>
+        </xsd:annotation>
+        <xsd:sequence>
+            <xsd:element name="identifier" type="xsd:string" minOccurs="0">
+                <xsd:annotation>
+                    <xsd:documentation>
+                        Counter identifier (key).
+                    </xsd:documentation>
+                </xsd:annotation>
+            </xsd:element>
+            <xsd:element name="value" type="xsd:int" minOccurs="0">
+                <xsd:annotation>
+                    <xsd:documentation>
+                        Current counter value.
+                    </xsd:documentation>
+                </xsd:annotation>
+            </xsd:element>
+            <!-- TODO other data, like start timestamp (maybe) -->
+        </xsd:sequence>
+        <xsd:attribute name="id" type="xsd:long"/>
+    </xsd:complexType>
 </xsd:schema>

model/certification-impl/src/main/java/com/evolveum/midpoint/certification/impl/CertificationHook.java

@@ -6,15 +6,13 @@
  */
 package com.evolveum.midpoint.certification.impl;
 
-import com.evolveum.midpoint.model.api.context.EvaluatedAssignment;
 import com.evolveum.midpoint.model.api.context.EvaluatedPolicyRule;
 import com.evolveum.midpoint.model.api.context.ModelContext;
 import com.evolveum.midpoint.model.api.context.ModelState;
 import com.evolveum.midpoint.model.api.hooks.ChangeHook;
 import com.evolveum.midpoint.model.api.hooks.HookOperationMode;
 import com.evolveum.midpoint.model.api.hooks.HookRegistry;
 import com.evolveum.midpoint.model.impl.lens.LensElementContext;
-import com.evolveum.midpoint.prism.delta.DeltaSetTriple;
 import com.evolveum.midpoint.schema.constants.SchemaConstants;
 import com.evolveum.midpoint.schema.result.OperationResult;
 import com.evolveum.midpoint.task.api.Task;
@@ -32,7 +30,6 @@
 import javax.annotation.PostConstruct;
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.List;
 import java.util.stream.Collectors;
 
@@ -79,7 +76,7 @@ public <O extends ObjectType> HookOperationMode invoke(@NotNull ModelContext<O>
     }
 
     private Collection<CertificationPolicyActionType> getFocusCertificationActions(ModelContext<?> context) {
-        return getCertificationActions(context.getFocusContext().getPolicyRules());
+        return getCertificationActions(context.getFocusContext().getObjectPolicyRules());
     }
 
     private Collection<CertificationPolicyActionType> getAssignmentCertificationActions(ModelContext<?> context) {

model/model-api/src/main/java/com/evolveum/midpoint/model/api/ModelInteractionService.java

@@ -465,8 +465,11 @@ TaskType submitTaskFromTemplate(String templateTaskOid, Map<QName, Object> exten
      */
     @Experimental
     @NotNull
-    Collection<EvaluatedPolicyRule> evaluateCollectionPolicyRules(@NotNull PrismObject<ObjectCollectionType> collection, @Nullable CompiledObjectCollectionView collectionView, @Nullable Class<? extends ObjectType> targetTypeClass, @NotNull Task task, @NotNull OperationResult result)
-            throws ObjectNotFoundException, SchemaException, SecurityViolationException, CommunicationException, ConfigurationException, ExpressionEvaluationException;
+    Collection<EvaluatedPolicyRule> evaluateCollectionPolicyRules(@NotNull PrismObject<ObjectCollectionType> collection,
+            @Nullable CompiledObjectCollectionView collectionView, @Nullable Class<? extends ObjectType> targetTypeClass,
+            @NotNull Task task, @NotNull OperationResult result)
+            throws ObjectNotFoundException, SchemaException, SecurityViolationException, CommunicationException,
+            ConfigurationException, ExpressionEvaluationException;
 
     @Experimental
     @NotNull

model/model-api/src/main/java/com/evolveum/midpoint/model/api/context/AssignmentPathSegment.java

@@ -49,6 +49,8 @@ public interface AssignmentPathSegment extends DebugDumpable, ShortDumpable, Ser
 
     ObjectType getSource();
 
+    String getSourceOid();
+
     ObjectType getTarget();
 
     QName getRelation();

model/model-api/src/main/java/com/evolveum/midpoint/model/api/context/EvaluatedCollectionStatsTrigger.java

@@ -23,7 +23,7 @@ public EvaluatedCollectionStatsTrigger(@NotNull PolicyConstraintKindType kind, @
     }
 
     @Override
-    public EvaluatedCollectionStatsTriggerType toEvaluatedPolicyRuleTriggerType(PolicyRuleExternalizationOptions options, PrismContext prismContext) {
+    public EvaluatedCollectionStatsTriggerType toEvaluatedPolicyRuleTriggerBean(PolicyRuleExternalizationOptions options, PrismContext prismContext) {
         EvaluatedCollectionStatsTriggerType rv = new EvaluatedCollectionStatsTriggerType();
         fillCommonContent(rv);
         return rv;

model/model-api/src/main/java/com/evolveum/midpoint/model/api/context/EvaluatedCompositeTrigger.java

@@ -41,7 +41,7 @@ public Collection<EvaluatedPolicyRuleTrigger<?>> getInnerTriggers() {
     public String toDiagShortcut() {
         return super.toDiagShortcut()
             + innerTriggers.stream()
-                    .map(trigger -> trigger.toDiagShortcut())
+                    .map(EvaluatedPolicyRuleTrigger::toDiagShortcut)
                     .distinct()
                     .collect(Collectors.joining("+", "(", ")"));
     }
@@ -69,12 +69,12 @@ protected void debugDumpSpecific(StringBuilder sb, int indent) {
     }
 
     @Override
-    public EvaluatedLogicalTriggerType toEvaluatedPolicyRuleTriggerType(PolicyRuleExternalizationOptions options,
+    public EvaluatedLogicalTriggerType toEvaluatedPolicyRuleTriggerBean(PolicyRuleExternalizationOptions options,
             PrismContext prismContext) {
         EvaluatedLogicalTriggerType rv = new EvaluatedLogicalTriggerType();
         fillCommonContent(rv);
         if (!options.isRespectFinalFlag() || !isFinal()) {
-            innerTriggers.forEach(t -> rv.getEmbedded().add(t.toEvaluatedPolicyRuleTriggerType(options, prismContext)));
+            innerTriggers.forEach(t -> rv.getEmbedded().add(t.toEvaluatedPolicyRuleTriggerBean(options, prismContext)));
         }
         return rv;
     }

model/model-api/src/main/java/com/evolveum/midpoint/model/api/context/EvaluatedExclusionTrigger.java

@@ -87,7 +87,7 @@ protected void debugDumpSpecific(StringBuilder sb, int indent) {
     }
 
     @Override
-    public EvaluatedExclusionTriggerType toEvaluatedPolicyRuleTriggerType(PolicyRuleExternalizationOptions options,
+    public EvaluatedExclusionTriggerType toEvaluatedPolicyRuleTriggerBean(PolicyRuleExternalizationOptions options,
             PrismContext prismContext) {
         EvaluatedExclusionTriggerType rv = new EvaluatedExclusionTriggerType();
         fillCommonContent(rv);

model/model-api/src/main/java/com/evolveum/midpoint/model/api/context/EvaluatedHasAssignmentTrigger.java

@@ -26,7 +26,7 @@ public EvaluatedHasAssignmentTrigger(@NotNull PolicyConstraintKindType kind, @No
     }
 
     @Override
-    public EvaluatedHasAssignmentTriggerType toEvaluatedPolicyRuleTriggerType(PolicyRuleExternalizationOptions options,
+    public EvaluatedHasAssignmentTriggerType toEvaluatedPolicyRuleTriggerBean(PolicyRuleExternalizationOptions options,
             PrismContext prismContext) {
         EvaluatedHasAssignmentTriggerType rv = new EvaluatedHasAssignmentTriggerType();
         fillCommonContent(rv);

model/model-api/src/main/java/com/evolveum/midpoint/model/api/context/EvaluatedModificationTrigger.java

@@ -30,7 +30,7 @@ public EvaluatedModificationTrigger(@NotNull PolicyConstraintKindType kind, @Not
     }
 
     @Override
-    public EvaluatedModificationTriggerType toEvaluatedPolicyRuleTriggerType(PolicyRuleExternalizationOptions options,
+    public EvaluatedModificationTriggerType toEvaluatedPolicyRuleTriggerBean(PolicyRuleExternalizationOptions options,
             PrismContext prismContext) {
         EvaluatedModificationTriggerType rv = new EvaluatedModificationTriggerType();
         fillCommonContent(rv);

model/model-api/src/main/java/com/evolveum/midpoint/model/api/context/EvaluatedMultiplicityTrigger.java

@@ -23,7 +23,7 @@ public EvaluatedMultiplicityTrigger(@NotNull PolicyConstraintKindType kind, @Not
     }
 
     @Override
-    public EvaluatedMultiplicityTriggerType toEvaluatedPolicyRuleTriggerType(PolicyRuleExternalizationOptions options,
+    public EvaluatedMultiplicityTriggerType toEvaluatedPolicyRuleTriggerBean(PolicyRuleExternalizationOptions options,
             PrismContext prismContext) {
         EvaluatedMultiplicityTriggerType rv = new EvaluatedMultiplicityTriggerType();
         fillCommonContent(rv);

model/model-api/src/main/java/com/evolveum/midpoint/model/api/context/EvaluatedPolicyRule.java

@@ -12,6 +12,7 @@
 import java.util.function.Predicate;
 
 import com.evolveum.midpoint.prism.PrismContext;
+import com.evolveum.midpoint.repo.api.Countable;
 import com.evolveum.midpoint.util.DebugDumpable;
 import com.evolveum.midpoint.util.LocalizableMessage;
 import com.evolveum.midpoint.util.TreeNode;
@@ -23,7 +24,7 @@
  * @author semancik
  *
  */
-public interface EvaluatedPolicyRule extends DebugDumpable, Serializable, Cloneable {
+public interface EvaluatedPolicyRule extends DebugDumpable, Serializable, Cloneable, Countable {
 
     @NotNull
     Collection<EvaluatedPolicyRuleTrigger<?>> getTriggers();
@@ -46,7 +47,7 @@ default boolean isTriggered() {
 
     String getName();
 
-    PolicyRuleType getPolicyRule();
+    @NotNull PolicyRuleType getPolicyRule();
 
     PolicyConstraintsType getPolicyConstraints();
 
@@ -68,7 +69,7 @@ default boolean isTriggered() {
 
     Collection<PolicyExceptionType> getPolicyExceptions();
 
-    void addToEvaluatedPolicyRuleTypes(Collection<EvaluatedPolicyRuleType> rules, PolicyRuleExternalizationOptions options,
+    void addToEvaluatedPolicyRuleBeans(Collection<EvaluatedPolicyRuleType> rules, PolicyRuleExternalizationOptions options,
             Predicate<EvaluatedPolicyRuleTrigger<?>> triggerSelector, PrismContext prismContext);
 
     boolean isGlobal();
@@ -97,4 +98,10 @@ void addToEvaluatedPolicyRuleTypes(Collection<EvaluatedPolicyRuleType> rules, Po
 
     //experimental
     String getPolicyRuleIdentifier();
+
+    default boolean hasThreshold() {
+        return getPolicyRule().getPolicyThreshold() != null; // refine this if needed
+    }
+
+    int getCount();
 }

model/model-api/src/main/java/com/evolveum/midpoint/model/api/context/EvaluatedPolicyRuleTrigger.java

@@ -139,7 +139,7 @@ public String toDiagShortcut() {
         return PolicyRuleTypeUtil.toDiagShortcut(constraintKind);
     }
 
-    public EvaluatedPolicyRuleTriggerType toEvaluatedPolicyRuleTriggerType(PolicyRuleExternalizationOptions options,
+    public EvaluatedPolicyRuleTriggerType toEvaluatedPolicyRuleTriggerBean(PolicyRuleExternalizationOptions options,
             PrismContext prismContext) {
         EvaluatedPolicyRuleTriggerType rv = new EvaluatedPolicyRuleTriggerType();
         fillCommonContent(rv);

model/model-api/src/main/java/com/evolveum/midpoint/model/api/context/EvaluatedSituationTrigger.java

@@ -95,12 +95,12 @@ protected void debugDumpSpecific(StringBuilder sb, int indent) {
     }
 
     @Override
-    public EvaluatedSituationTriggerType toEvaluatedPolicyRuleTriggerType(PolicyRuleExternalizationOptions options,
+    public EvaluatedSituationTriggerType toEvaluatedPolicyRuleTriggerBean(PolicyRuleExternalizationOptions options,
             PrismContext prismContext) {
         EvaluatedSituationTriggerType rv = new EvaluatedSituationTriggerType();
         fillCommonContent(rv);
         if (!options.isRespectFinalFlag() || !isFinal()) {
-            sourceRules.forEach(r -> r.addToEvaluatedPolicyRuleTypes(rv.getSourceRule(), options, null, prismContext));
+            sourceRules.forEach(r -> r.addToEvaluatedPolicyRuleBeans(rv.getSourceRule(), options, null, prismContext));
         }
         return rv;
     }

model/model-api/src/main/java/com/evolveum/midpoint/model/api/context/EvaluatedStateTrigger.java

@@ -25,7 +25,7 @@ public EvaluatedStateTrigger(@NotNull PolicyConstraintKindType kind, @NotNull St
     }
 
     @Override
-    public EvaluatedStateTriggerType toEvaluatedPolicyRuleTriggerType(PolicyRuleExternalizationOptions options,
+    public EvaluatedStateTriggerType toEvaluatedPolicyRuleTriggerBean(PolicyRuleExternalizationOptions options,
             PrismContext prismContext) {
         EvaluatedStateTriggerType rv = new EvaluatedStateTriggerType();
         fillCommonContent(rv);

model/model-api/src/main/java/com/evolveum/midpoint/model/api/context/EvaluatedTimeValidityTrigger.java

@@ -23,7 +23,7 @@ public EvaluatedTimeValidityTrigger(@NotNull PolicyConstraintKindType kind, @Not
     }
 
     @Override
-    public EvaluatedTimeValidityTriggerType toEvaluatedPolicyRuleTriggerType(PolicyRuleExternalizationOptions options,
+    public EvaluatedTimeValidityTriggerType toEvaluatedPolicyRuleTriggerBean(PolicyRuleExternalizationOptions options,
             PrismContext prismContext) {
         EvaluatedTimeValidityTriggerType rv = new EvaluatedTimeValidityTriggerType();
         fillCommonContent(rv);

model/model-api/src/main/java/com/evolveum/midpoint/model/api/context/EvaluatedTransitionTrigger.java

@@ -67,12 +67,12 @@ protected void debugDumpSpecific(StringBuilder sb, int indent) {
     }
 
     @Override
-    public EvaluatedTransitionTriggerType toEvaluatedPolicyRuleTriggerType(PolicyRuleExternalizationOptions options,
+    public EvaluatedTransitionTriggerType toEvaluatedPolicyRuleTriggerBean(PolicyRuleExternalizationOptions options,
             PrismContext prismContext) {
         EvaluatedTransitionTriggerType rv = new EvaluatedTransitionTriggerType();
         fillCommonContent(rv);
         if (!options.isRespectFinalFlag() || !isFinal()) {
-            innerTriggers.forEach(t -> rv.getEmbedded().add(t.toEvaluatedPolicyRuleTriggerType(options, prismContext)));
+            innerTriggers.forEach(t -> rv.getEmbedded().add(t.toEvaluatedPolicyRuleTriggerBean(options, prismContext)));
         }
         return rv;
     }

model/model-api/src/main/java/com/evolveum/midpoint/model/api/context/ModelElementContext.java

@@ -78,8 +78,7 @@ public interface ModelElementContext<O extends ObjectType> extends Serializable,
      * Returns all policy rules that apply to this object - even those that were not triggered.
      * The policy rules are compiled from all the applicable sources (target, meta-roles, etc.)
      */
-    @NotNull
-    Collection<? extends EvaluatedPolicyRule> getPolicyRules();
+    @NotNull Collection<? extends EvaluatedPolicyRule> getObjectPolicyRules();
 
     /**
      * Initial intent regarding the account. It indicated what the initiator of the operation WANTS TO DO with the