Eliminate repeated delta/object recomputations

Each time current or summary delta (including relative and absolute ODO)
were obtained from LensElementContext, full re-creation of these deltas
from primary and secondary deltas occurred. This accounted for large
portion of time spent in the clockwork for some use cases.

This commit partly eliminates these recomputations: the computable
parts of the element state (mainly: current/summary delta, new object)
are cached, with invalidation bound to the changes in "source" data.

Work in progress. The performance can be improved further
by updating current/summary delta "on the fly", along with updates
to secondary deltas. It is also questionable if it is better to
recompute new object when accessed (and invalid), or to use original
approach based on LensElementContext#recompute method, or to update
it "on the fly" along with current/summary delta.

However, even this partial work saved 10-15% clockwork time
for scenarios with 100 assignments, and 30-35% time for 500 assignments.

Unrelated minor changes:

1. ConsolidationProcessor no longer drops primary and secondary deltas
for projections that were deleted.

2. Synchronization intent was moved from LensElementContext
to LensProjectionContext.

Related to MID-7053.

infra/schema/src/main/java/com/evolveum/midpoint/schema/ObjectTreeDeltas.java

@@ -109,7 +109,7 @@ public ObjectTreeDeltas<T> clone() {
         return clone;
     }
 
-    public Set<? extends Map.Entry<ResourceShadowDiscriminator, ObjectDelta<ShadowType>>> getProjectionChangeMapEntries() {
+    public Set<Map.Entry<ResourceShadowDiscriminator, ObjectDelta<ShadowType>>> getProjectionChangeMapEntries() {
         return projectionChangeMap.entrySet();
     }
 
@@ -123,9 +123,7 @@ private ObjectTreeDeltasType toObjectTreeDeltasType() throws SchemaException {
         if (getFocusChange() != null) {
             jaxb.setFocusPrimaryDelta(DeltaConvertor.toObjectDeltaType(getFocusChange()));
         }
-        //noinspection unchecked
-        Set<Map.Entry<ResourceShadowDiscriminator, ObjectDelta<ShadowType>>> entries =
-                (Set<Map.Entry<ResourceShadowDiscriminator, ObjectDelta<ShadowType>>>) getProjectionChangeMapEntries();
+        Set<Map.Entry<ResourceShadowDiscriminator, ObjectDelta<ShadowType>>> entries = getProjectionChangeMapEntries();
         for (Map.Entry<ResourceShadowDiscriminator, ObjectDelta<ShadowType>> entry : entries) {
             ProjectionObjectDeltaType projChange = new ProjectionObjectDeltaType();
             projChange.setResourceShadowDiscriminator(entry.getKey().toResourceShadowDiscriminatorType());

infra/schema/src/main/java/com/evolveum/midpoint/schema/ResourceShadowDiscriminator.java

@@ -42,10 +42,10 @@ public class ResourceShadowDiscriminator implements Serializable, DebugDumpable,
 
     private static final long serialVersionUID = 346600684011645741L;
 
-    private String resourceOid;
+    private final String resourceOid;
     private ShadowKindType kind = ShadowKindType.ACCOUNT;
     private String intent;
-    private String tag;
+    private final String tag;
     private QName objectClass;
     private boolean tombstone;
     private int order = 0;
@@ -75,39 +75,38 @@ public ResourceShadowDiscriminator(ShadowDiscriminatorType accRefType, String de
         this.tombstone = false;
         setIntent(accRefType.getIntent());
         setKind(kind);
+        this.tag = null;
     }
 
     public ResourceShadowDiscriminator(String resourceOid) {
         this.resourceOid = resourceOid;
+        this.tag = null;
     }
 
     public ResourceShadowDiscriminator(String resourceOid, QName objectClass) {
         this.resourceOid = resourceOid;
         this.objectClass = objectClass;
         this.kind = null;
+        this.tag = null;
     }
 
     public String getResourceOid() {
         return resourceOid;
     }
 
-    public void setResourceOid(String resourceOid) {
-        this.resourceOid = resourceOid;
-    }
-
     public ShadowKindType getKind() {
         return kind;
     }
 
-    public void setKind(ShadowKindType kind) {
+    private void setKind(ShadowKindType kind) {
         this.kind = kind;
     }
 
     public String getIntent() {
         return intent;
     }
 
-    public void setIntent(String intent) {
+    private void setIntent(String intent) {
 //        if (intent == null) {
 //            intent = SchemaConstants.INTENT_DEFAULT;
 //        }
@@ -118,10 +117,6 @@ public String getTag() {
         return tag;
     }
 
-    public void setTag(String tag) {
-        this.tag = tag;
-    }
-
     public QName getObjectClass() {
         return objectClass;
     }

model/certification-impl/src/main/java/com/evolveum/midpoint/certification/impl/AccCertTimedActionTriggerHandler.java

@@ -53,7 +53,8 @@ private void initialize() {
     }
 
     @Override
-    public <O extends ObjectType> void handle(PrismObject<O> object, TriggerType trigger, RunningTask triggerScannerTask, OperationResult parentResult) {
+    public <O extends ObjectType> void handle(@NotNull PrismObject<O> object, @NotNull TriggerType trigger,
+            @NotNull RunningTask triggerScannerTask, @NotNull OperationResult parentResult) {
         if (!(object.asObjectable() instanceof AccessCertificationCampaignType)) {
             throw new IllegalArgumentException("Unexpected object type: should be AccessCertificationCampaignType: " + object);
         }

model/certification-impl/src/main/java/com/evolveum/midpoint/certification/impl/AccessCertificationCampaignReiterationTriggerHandler.java

@@ -22,6 +22,7 @@
 import com.evolveum.midpoint.xml.ns._public.common.common_3.AccessCertificationCampaignType;
 import com.evolveum.midpoint.xml.ns._public.common.common_3.ObjectType;
 import com.evolveum.midpoint.xml.ns._public.common.common_3.TriggerType;
+import org.jetbrains.annotations.NotNull;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
@@ -47,7 +48,8 @@ private void initialize() {
     }
 
     @Override
-    public <O extends ObjectType> void handle(PrismObject<O> prismObject, TriggerType trigger, RunningTask task, OperationResult result) {
+    public <O extends ObjectType> void handle(@NotNull PrismObject<O> prismObject, @NotNull TriggerType trigger,
+            @NotNull RunningTask task, @NotNull OperationResult result) {
         try {
             ObjectType object = prismObject.asObjectable();
             if (!(object instanceof AccessCertificationCampaignType)) {

model/certification-impl/src/main/java/com/evolveum/midpoint/certification/impl/AccessCertificationCloseStageApproachingTriggerHandler.java

@@ -20,6 +20,7 @@
 import com.evolveum.midpoint.util.logging.Trace;
 import com.evolveum.midpoint.util.logging.TraceManager;
 import com.evolveum.midpoint.xml.ns._public.common.common_3.*;
+import org.jetbrains.annotations.NotNull;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
@@ -51,7 +52,8 @@ private void initialize() {
     }
 
     @Override
-    public <O extends ObjectType> void handle(PrismObject<O> prismObject, TriggerType trigger, RunningTask task, OperationResult result) {
+    public <O extends ObjectType> void handle(@NotNull PrismObject<O> prismObject, @NotNull TriggerType trigger,
+            @NotNull RunningTask task, @NotNull OperationResult result) {
         try {
             ObjectType object = prismObject.asObjectable();
             if (!(object instanceof AccessCertificationCampaignType)) {

model/certification-impl/src/main/java/com/evolveum/midpoint/certification/impl/AccessCertificationCloseStageTriggerHandler.java

@@ -21,6 +21,7 @@
 import com.evolveum.midpoint.xml.ns._public.common.common_3.AccessCertificationCampaignType;
 import com.evolveum.midpoint.xml.ns._public.common.common_3.ObjectType;
 import com.evolveum.midpoint.xml.ns._public.common.common_3.TriggerType;
+import org.jetbrains.annotations.NotNull;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
@@ -48,7 +49,8 @@ private void initialize() {
     }
 
     @Override
-    public <O extends ObjectType> void handle(PrismObject<O> prismObject, TriggerType trigger, RunningTask task, OperationResult result) {
+    public <O extends ObjectType> void handle(@NotNull PrismObject<O> prismObject, @NotNull TriggerType trigger,
+            @NotNull RunningTask task, @NotNull OperationResult result) {
         try {
             ObjectType object = prismObject.asObjectable();
             if (!(object instanceof AccessCertificationCampaignType)) {

model/model-api/src/main/java/com/evolveum/midpoint/model/api/context/ModelElementContext.java

@@ -81,20 +81,13 @@ public interface ModelElementContext<O extends ObjectType> extends Serializable,
     @NotNull Collection<? extends EvaluatedPolicyRule> getObjectPolicyRules();
 
     /**
-     * Initial intent regarding the account. It indicated what the initiator of the operation WANTS TO DO with the
-     * context.
-     * If set to null then the decision is left to "the engine". Null is also a typical value
-     * when the context is created. It may be pre-set under some circumstances, e.g. if an account is being unlinked.
-     */
-    SynchronizationIntent getSynchronizationIntent();
-
-    /**
-     * @return true if the object (focus or projection) is to be added
+     * Returns true if the object (focus or projection) is to be added.
      */
     boolean isAdd();
 
     /**
-     * @return true if the object (focus or projection) is to be deleted
+     * Returns true if the object (focus or projection) is to be deleted. This is determined
+     * solely by looking at caller's intents (primary delta, sync delta, and/or sync intent).
      */
     boolean isDelete();
 
@@ -104,14 +97,14 @@ public interface ModelElementContext<O extends ObjectType> extends Serializable,
     ObjectDelta<O> getPrimaryDelta();
 
     /**
-     * Sets the primary delta. Not to be publicly used. TODO reconsider this method here.
+     * Sets the primary delta. Must be called before the clockwork starts.
      */
     void setPrimaryDelta(ObjectDelta<O> primaryDelta);
 
     /**
-     * Add a delta to the primary delta. Not to be publicly used. TODO reconsider this method here.
+     * Add a delta to the primary delta. Must be called before the clockwork starts.
      */
-    void addPrimaryDelta(ObjectDelta<O> value) throws SchemaException;
+    void addToPrimaryDelta(ObjectDelta<O> value) throws SchemaException;
 
     /**
      * Returns secondary delta for the current clockwork click.
@@ -137,6 +130,8 @@ public interface ModelElementContext<O extends ObjectType> extends Serializable,
     /**
      * Returns all secondary deltas, merged together.
      *
+     * (Can take some time to compute. So use with care.)
+     *
      * The returned object is (kind of) immutable. Changing it may do strange things, but most likely the changes will be lost.
      */
     ObjectDelta<O> getSummarySecondaryDelta();

model/model-api/src/main/java/com/evolveum/midpoint/model/api/context/ModelProjectionContext.java

@@ -1,51 +1,65 @@
-/*
- * Copyright (c) 2010-2017 Evolveum and contributors
- *
- * This work is dual-licensed under the Apache License 2.0
- * and European Union Public License. See LICENSE file for details.
- */
-package com.evolveum.midpoint.model.api.context;
-
-import com.evolveum.midpoint.prism.delta.ObjectDelta;
-import com.evolveum.midpoint.schema.ResourceShadowDiscriminator;
-import com.evolveum.midpoint.util.exception.SchemaException;
-import com.evolveum.midpoint.xml.ns._public.common.common_3.ShadowType;
-
-/**
- * @author semancik
- *
- */
-public interface ModelProjectionContext extends ModelElementContext<ShadowType> {
-
-    /**
-     * Returns synchronization delta.
-     *
-     * Synchronization delta describes changes that have recently happened. MidPoint reacts to these
-     * changes by "pulling them in" (e.g. using them in inbound mappings).
-     */
-    ObjectDelta<ShadowType> getSyncDelta();
-
-    void setSyncDelta(ObjectDelta<ShadowType> syncDelta);
-
-    ResourceShadowDiscriminator getResourceShadowDiscriminator();
-
-    /**
-     * Decision regarding the account. It describes the overall situation of the account e.g. whether account
-     * is added, is to be deleted, unliked, etc.
-     *
-     * If set to null no decision was made yet. Null is also a typical value when the context is created.
-     *
-     * @see SynchronizationPolicyDecision
-     */
-    SynchronizationPolicyDecision getSynchronizationPolicyDecision();
-
-    ObjectDelta<ShadowType> getExecutableDelta() throws SchemaException;
-
-    boolean isFullShadow();
-
-    Boolean isLegal();
-
-    boolean isExists();
-
-    boolean isTombstone();
-}
+/*
+ * Copyright (c) 2010-2017 Evolveum and contributors
+ *
+ * This work is dual-licensed under the Apache License 2.0
+ * and European Union Public License. See LICENSE file for details.
+ */
+package com.evolveum.midpoint.model.api.context;
+
+import com.evolveum.midpoint.prism.delta.ObjectDelta;
+import com.evolveum.midpoint.schema.ResourceShadowDiscriminator;
+import com.evolveum.midpoint.util.exception.SchemaException;
+import com.evolveum.midpoint.xml.ns._public.common.common_3.ShadowType;
+
+/**
+ * @author semancik
+ *
+ */
+public interface ModelProjectionContext extends ModelElementContext<ShadowType> {
+
+    /**
+     * Returns synchronization delta.
+     *
+     * Synchronization delta describes changes that have recently happened. MidPoint reacts to these
+     * changes by "pulling them in" (e.g. using them in inbound mappings).
+     */
+    ObjectDelta<ShadowType> getSyncDelta();
+
+    void setSyncDelta(ObjectDelta<ShadowType> syncDelta);
+
+    ResourceShadowDiscriminator getResourceShadowDiscriminator();
+
+    /**
+     * Initial intent regarding the account. It indicates what the initiator of the operation
+     * _wants to do_ with the context.
+     *
+     * If set to null then the decision is left to "the engine". Null is also a typical value
+     * when the context is created. It may be pre-set under some circumstances, e.g. if an account is being unlinked.
+     */
+    SynchronizationIntent getSynchronizationIntent();
+
+    /**
+     * Decision regarding the account. It describes the overall situation of the account e.g. whether account
+     * is added, is to be deleted, unliked, etc.
+     *
+     * If set to null no decision was made yet. Null is also a typical value when the context is created.
+     *
+     * @see SynchronizationPolicyDecision
+     */
+    SynchronizationPolicyDecision getSynchronizationPolicyDecision();
+
+    /**
+     * Returns delta suitable for execution. The primary and secondary deltas may not make complete sense all by themselves.
+     * E.g. they may both be MODIFY deltas even in case that the account should be created. The deltas begin to make sense
+     * only if combined with sync decision. This method provides the deltas all combined and ready for execution.
+     */
+    ObjectDelta<ShadowType> getExecutableDelta() throws SchemaException;
+
+    boolean isFullShadow();
+
+    Boolean isLegal();
+
+    boolean isExists();
+
+    boolean isTombstone();
+}

model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/controller/ModelController.java

@@ -13,6 +13,7 @@
 
 import java.io.*;
 import java.util.*;
+import java.util.Objects;
 import javax.xml.namespace.QName;
 
 import com.evolveum.midpoint.repo.common.activity.TaskActivityManager;
@@ -436,11 +437,13 @@ private Collection<ObjectDeltaOperation<? extends ObjectType>> executeChangesNon
             }
 
             if (context.hasExplosiveProjection()) {
-                PrismObject<? extends ObjectType> focus = context.getFocusContext().getObjectAny();
+                PrismObject<? extends ObjectType> focus = Objects.requireNonNull(
+                        context.getFocusContext().getObjectAny(), "no focus object");
 
                 LOGGER.debug("Recomputing {} because there was explosive projection", focus);
 
-                LensContext<? extends ObjectType> recomputeContext = contextFactory.createRecomputeContext(focus, options, task, result);
+                LensContext<? extends ObjectType> recomputeContext =
+                        contextFactory.createRecomputeContext(focus, options, task, result);
                 recomputeContext.setDoReconciliationForAllProjections(true);
                 LOGGER.trace("Recomputing {}, context:\n{}", focus, recomputeContext.debugDumpLazily());
                 clockwork.run(recomputeContext, task, result);

model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/lens/Clockwork.java

@@ -133,6 +133,7 @@ <F extends ObjectType> HookOperationMode runWithConflictDetection(LensContext<F>
             throws SchemaException, PolicyViolationException, ExpressionEvaluationException, ObjectNotFoundException,
             ObjectAlreadyExistsException, CommunicationException, ConfigurationException, SecurityViolationException {
 
+        context.setStarted();
         context.updateSystemConfiguration(result);
 
         LOGGER.trace("Running clockwork for context {}", context);

model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/lens/ClockworkClick.java

@@ -234,8 +234,7 @@ private void processSecondary(OperationResult result, OperationResult overallRes
 
         beans.clockworkAuditHelper.audit(context, AuditEventStage.EXECUTION, task, result, overallResult);
 
-        context.rotIfNeeded();
-        context.resetDeltasAfterExecution();
+        context.updateAfterExecution();
 
         if (!restartRequestedHolder.getValue()) {
             context.incrementExecutionWave();
@@ -294,7 +293,7 @@ private void processClockworkException(Throwable e, OperationResult result, Oper
         result.recordFatalErrorNotFinish(e);
         beans.clockworkAuditHelper.auditEvent(context, AuditEventStage.EXECUTION, null, true, task, result, overallResult);
 
-        reclaimSequencesIfPossible(context, task, result);
+        reclaimSequencesIfPossible(result);
         result.recordEnd();
     }
 
@@ -306,7 +305,7 @@ private void processClockworkException(Throwable e, OperationResult result, Oper
      * solution (because sequence values were maybe not used in these deltas), but we have nothing
      * better at hand now.
      */
-    private <F extends ObjectType> void reclaimSequencesIfPossible(LensContext<F> context, Task task, OperationResult result) throws SchemaException {
+    private void reclaimSequencesIfPossible(OperationResult result) throws SchemaException {
         if (!context.wasAnythingExecuted()) {
             LensUtil.reclaimSequences(context, beans.cacheRepositoryService, task, result);
         } else {