Reload data-dependent projection before its wave

If there is a data dependency between projection (like an AD group) and
its dependee (like an OU in which the group resides), and the dependee
changes, we will reload the dependent projection on the start of its
wave.

This resolves MID-8929.

Note it's a preliminary fix, as it e.g. does not discriminate between
relevant and irrelevant changes. Also, maybe it would be sufficient
to load the full shadow ONLY at the beginning of the wave related to
the dependent projection. This can be optimized later. See MID-9083.

infra/schema/src/main/java/com/evolveum/midpoint/schema/processor/AbstractResourceObjectDefinitionImpl.java

@@ -134,7 +134,7 @@ public abstract class AbstractResourceObjectDefinitionImpl
      * refinements).
      * - For object type definitions, it is relevant `objectType` value in `schemaHandling`, expanded by resolving
      * object type inheritance. Note that object type definition that refers to refined object class definition is also
-     * a case of inheritance! Such definition bean contains all data from the refining object class definition bean.
+     * a case of inheritance! Such definition bean contains all data from the refined object class definition bean.
      *
      * Immutable.
      */

infra/schema/src/main/java/com/evolveum/midpoint/schema/processor/ResourceObjectDefinition.java

@@ -419,9 +419,6 @@ default ResourceAttributeContainer instantiate(ItemName itemName) {
     //region Other
     /**
      * Returns the "raw" configuration bean for this object type.
-     *
-     * BEWARE: In the case of inherited object types, this is only the partial information.
-     * (Parts inherited from the parents are not returned.)
      */
     @NotNull ResourceObjectTypeDefinitionType getDefinitionBean();
 

infra/schema/src/main/java/com/evolveum/midpoint/schema/processor/deleg/ResourceObjectDefinitionDelegator.java

@@ -6,7 +6,6 @@
 
 import com.evolveum.midpoint.schema.processor.ResourceAssociationDefinition;
 import com.evolveum.midpoint.prism.*;
-import com.evolveum.midpoint.prism.path.ItemName;
 import com.evolveum.midpoint.schema.processor.*;
 
 import com.evolveum.midpoint.xml.ns._public.common.common_3.*;

infra/schema/src/main/java/com/evolveum/midpoint/schema/util/ResourceObjectTypeDependencyTypeUtil.java

@@ -8,6 +8,7 @@
 package com.evolveum.midpoint.schema.util;
 
 import com.evolveum.midpoint.util.MiscUtil;
+import com.evolveum.midpoint.xml.ns._public.common.common_3.ResourceObjectTypeDependencyDataBindingKindType;
 import com.evolveum.midpoint.xml.ns._public.common.common_3.ResourceObjectTypeDependencyStrictnessType;
 import com.evolveum.midpoint.xml.ns._public.common.common_3.ResourceObjectTypeDependencyType;
 
@@ -67,4 +68,8 @@ public static boolean isStrict(ResourceObjectTypeDependencyType dependency) {
     public static boolean isForceLoadDependentShadow(ResourceObjectTypeDependencyType dependency) {
         return Boolean.TRUE.equals(dependency.isForceLoad());
     }
+
+    public static boolean isDataBindingPresent(ResourceObjectTypeDependencyType dependency) {
+        return dependency.getDataBinding() == ResourceObjectTypeDependencyDataBindingKindType.SOME;
+    }
 }

infra/schema/src/main/resources/xml/ns/public/common/common-provisioning-3.xsd

@@ -1685,6 +1685,13 @@
         <xsd:annotation>
             <xsd:documentation>
                 Describes a dependency of an object type on another object type.
+                For example, a role may have two Active Directory projections: an OU and a group residing in that OU.
+                We say that the group depends on the OU: the OU has to be provisioned first; and only after that, the group
+                can be created. When de-provisioning, the group has to be deleted first, and the OU only after that.
+
+                We call the group the "dependent object" or "depender". The OU is a "dependee".
+
+                This data structure has to reside within the dependent object (depender).
             </xsd:documentation>
             <xsd:appinfo>
                 <a:container/>
@@ -1737,6 +1744,22 @@
                     </xsd:documentation>
                 </xsd:annotation>
             </xsd:element>
+            <xsd:element name="dataBinding" type="tns:ResourceObjectTypeDependencyDataBindingKindType" minOccurs="0"
+                    default="none">
+                <xsd:annotation>
+                    <xsd:documentation>
+                        Are the attributes of the dependent object bound to the dependee attributes in any way?
+                        For example, in AD, if the distinguished name of an OU (dependee) changes, the DNs of all dependent
+                        objects (e.g. groups within that OU) change as well.
+
+                        EXPERIMENTAL. Mainly to implement MID-8929. Maybe it will not be needed in the future.
+                    </xsd:documentation>
+                    <xsd:appinfo>
+                        <a:since>4.8</a:since>
+                        <a:experimental>true</a:experimental>
+                    </xsd:appinfo>
+                </xsd:annotation>
+            </xsd:element>
         </xsd:sequence>
     </xsd:complexType>
 
@@ -1801,6 +1824,43 @@
         </xsd:restriction>
     </xsd:simpleType>
 
+    <xsd:simpleType name="ResourceObjectTypeDependencyDataBindingKindType">
+        <xsd:annotation>
+            <xsd:documentation>
+                Level of binding between dependent resource objects.
+            </xsd:documentation>
+            <xsd:appinfo>
+                <a:since>4.8</a:since>
+                <a:experimental>true</a:experimental>
+                <jaxb:typesafeEnumClass/>
+            </xsd:appinfo>
+        </xsd:annotation>
+        <xsd:restriction base="xsd:string">
+            <xsd:enumeration value="none">
+                <xsd:annotation>
+                    <xsd:documentation>
+                        Attributes of the dependent resource objects are not bound in any way.
+                    </xsd:documentation>
+                    <xsd:appinfo>
+                        <jaxb:typesafeEnumMember name="NONE"/>
+                    </xsd:appinfo>
+                </xsd:annotation>
+            </xsd:enumeration>
+            <xsd:enumeration value="some">
+                <xsd:annotation>
+                    <xsd:documentation>
+                        There is some binding between attributes of the dependent resource objects, but the exact character
+                        of this binding is not known or specified. We must assume that anything can happen when the dependee
+                        object changes.
+                    </xsd:documentation>
+                    <xsd:appinfo>
+                        <jaxb:typesafeEnumMember name="SOME"/>
+                    </xsd:appinfo>
+                </xsd:annotation>
+            </xsd:enumeration>
+        </xsd:restriction>
+    </xsd:simpleType>
+
     <xsd:complexType name="ResourceItemDefinitionType">
         <xsd:annotation>
             <xsd:documentation>

model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/lens/LensProjectionContext.java

@@ -338,7 +338,7 @@ private LensProjectionContext(
      * (Meaning that the source that contains `dependency` among its configured dependencies depends on `target`,
      * i.e. the `dependency` configuration points to `target`.)
      *
-     * Precondition: dependency is fully specified (resource, kind, intent are not null).
+     * Precondition: dependency is fully specified (resource, kind, intent are not null), see {@link #getDependencies()}.
      */
     public boolean isDependencyTarget(
             ResourceObjectTypeDependencyType dependency) {
@@ -1760,7 +1760,17 @@ public String getDescription() {
     @Experimental
     public boolean isCurrentForProjection() {
         return !completed
-                && (wave == -1 || wave == getLensContext().getProjectionWave());
+                && (!hasProjectionWave() || isCurrentProjectionWave());
+    }
+
+    public boolean hasProjectionWave() {
+        return wave != -1;
+    }
+
+    /** Assumes the wave is computed. */
+    public boolean isCurrentProjectionWave() {
+        stateCheck(hasProjectionWave(), "Projection wave was not yet determined for %s", this);
+        return wave == getLensContext().getProjectionWave();
     }
 
     public boolean isCompleted() {
@@ -2037,4 +2047,39 @@ public boolean isOutboundSyncDisabled(OperationResult result) {
         }
         return !policy.getSynchronize().getOutbound().isEnabled();
     }
+
+    /**
+     * Returns the projection contexts bound to this one via data dependency that are known (or supposed) to be modified.
+     */
+    public List<LensProjectionContext> getModifiedDataBoundDependees() throws SchemaException, ConfigurationException {
+        List<LensProjectionContext> matching = new ArrayList<>();
+        for (ResourceObjectTypeDependencyType dependencyBean : getDependencies()) {
+            if (ResourceObjectTypeDependencyTypeUtil.isDataBindingPresent(dependencyBean)) {
+                lensContext.getProjectionContexts().stream()
+                        .filter(ctx -> ctx != this)
+                        .filter(ctx -> ctx.isDependencyTarget(dependencyBean))
+                        .filter(ctx -> ctx.hasOnResourceModificationAttempt())
+                        .forEach(matching::add);
+            }
+        }
+        return matching;
+    }
+
+    /**
+     * Returns true if there was an operation (or attempted operation) dealing with the on-resource state.
+     *
+     * We consider ADD and DELETE operations as such, mainly to be sure.
+     * For MODIFY operations we check the individual modifications.
+     */
+    private boolean hasOnResourceModificationAttempt() {
+        for (LensObjectDeltaOperation<ShadowType> odo : getExecutedDeltas()) {
+            var delta = odo.getObjectDelta();
+            if (ObjectDelta.isAdd(delta)
+                    || ObjectDelta.isDelete(delta)
+                    || ObjectDelta.isModify(delta) && ShadowUtil.hasResourceModifications(delta.getModifications())) {
+                return true;
+            }
+        }
+        return false;
+    }
 }

model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/lens/projector/DependencyProcessor.java

@@ -521,7 +521,7 @@ void preprocessDependencies(LensContext<?> context) throws SchemaException, Conf
             for (ResourceObjectTypeDependencyType dependency : currentCtx.getDependencies()) {
                 ResourceObjectTypeDependencyStrictnessType strictness = getDependencyStrictness(dependency);
                 if (isForceLoadDependentShadow(dependency)
-                        && (strictness == STRICT || strictness == RELAXED)) {
+                        && (strictness == STRICT || strictness == RELAXED)) { // TODO why not when LAX?
                     // Before 4.6, we used RSD to match the contexts. It matched e.g. only contexts with order = 0.
                     // This one should be more appropriate.
                     LensProjectionContext upstreamCtx = findFirstUpstreamContext(currentCtx, dependency);

model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/lens/projector/loader/ProjectionUpdateOperation.java

@@ -34,6 +34,7 @@
 import org.jetbrains.annotations.NotNull;
 
 import java.util.Collection;
+import java.util.List;
 
 import static com.evolveum.midpoint.prism.PrismObject.asObjectable;
 import static com.evolveum.midpoint.schema.GetOperationOptions.isNoFetch;
@@ -42,8 +43,7 @@
  * Updates the projection context:
  *
  * . Sets the "do reconciliation" flag for volatile resources.
- * . Loads the object (from repo or from resource), if needed. See {@link #loadCurrentObjectIfNeeded(OperationResult)}
- * and {@link #needToReload()}.
+ * . Loads the object (from repo or from resource), if needed. See {@link #loadCurrentObjectIfNeeded(OperationResult)}.
  * . Loads the resource, if not loaded yet.
  * . Sets projection security policy.
  * . Sets "can project" flag if limited propagation option is present.
@@ -134,18 +134,19 @@ private void updateInternal(OperationResult result)
     }
 
     /**
-     * Loads the current object, if it's not loaded or if it needs to be reloaded.
+     * Loads the current object, if needed. See {@link #shouldLoadCurrentObject()} for the exact algorithm.
      *
      * Returns true if an error occurred.
      */
     private boolean loadCurrentObjectIfNeeded(OperationResult result)
             throws SchemaException, ExpressionEvaluationException, CommunicationException, ConfigurationException,
             ObjectNotFoundException, SecurityViolationException {
+
         projectionObject = asObjectable(projectionContext.getObjectCurrent());
-        if (projectionContext.getObjectCurrent() == null || needToReload()) {
+
+        if (shouldLoadCurrentObject()) {
             return loadCurrentObject(result);
         } else {
-            LOGGER.trace("No need to reload the object");
             if (projectionObjectOid != null) {
                 projectionContext.setExists(
                         ShadowUtil.isExists(projectionObject));
@@ -154,6 +155,65 @@ private boolean loadCurrentObjectIfNeeded(OperationResult result)
         }
     }
 
+    /**
+     * Should the object be loaded or reloaded?
+     *
+     * Coupled with {@link #createProjectionLoadingOptions()} regarding whether `noFetch` option should be used.
+     *
+     * There is an interesting side effect of "no fetch" loading of already-loaded object: the "full shadow" flag is discarded
+     * in such cases. This may ensure the consistency at the cost of resource object re-loading.
+     */
+    private boolean shouldLoadCurrentObject() throws SchemaException, ConfigurationException {
+        if (projectionContext.getObjectCurrent() == null) {
+            LOGGER.trace("Will load current object, as there is none loaded");
+            return true;
+        }
+
+        if (projectionContext.isDoReconciliation() && !projectionContext.isFullShadow()) {
+            LOGGER.trace("Will reload current object, because we are doing reconciliation and we do not have full shadow");
+            return true; // Note that the loading options will ensure that the full object is loaded.
+        }
+
+        // This is kind of brutal. But effective. We are reloading all higher-order dependencies
+        // before they are processed. This makes sure we have fresh state when they are re-computed.
+        // Because higher-order dependencies may have more than one projection context and the
+        // changes applied to one of them are not automatically reflected on on other. therefore we need to reload.
+        //
+        // Note: we can safely assume that the projection wave is known, as the order is > 0
+        // (order is determined by the dependency processor).
+        if (projectionContext.getOrder() > 0
+                && projectionContext.isCurrentProjectionWave()) {
+            LOGGER.trace("Will reload higher-order context because its wave has come (projection ctx wave = {})",
+                    projectionContext.getWave());
+            return true;
+        }
+
+        List<LensProjectionContext> modifiedDependees = projectionContext.getModifiedDataBoundDependees();
+        if (!modifiedDependees.isEmpty()
+                && projectionContext.hasProjectionWave()
+                && projectionContext.isCurrentProjectionWave()) {
+            // Reloading the projection if some of its data-dependees changed (and if it's wave has come). See MID-8929.
+            // We do not reload if the wave is not known. This is to avoid useless reloading at the very beginning.
+            //
+            // In the future, we may consider optimizing the loading by removing the initial loading of these projections.
+            // See MID-9083.
+            LOGGER.trace(
+                    "Will reload context with modified data-bound dependee because its wave has come. "
+                            + "Projection ctx wave = {}, modified data-bound dependees = {}",
+                    projectionContext.getWave(), modifiedDependees);
+            return true;
+        }
+
+        LOGGER.trace("No explicit reason for reloading current object "
+                        + "(recon: {}, full: {}, order: {}, wave: {}, modified deps: {})",
+                projectionContext.isDoReconciliation(),
+                projectionContext.isFullShadow(),
+                projectionContext.getOrder(),
+                projectionContext.getWave(),
+                modifiedDependees);
+        return false;
+    }
+
     /**
      * If "limit propagation" option is set, we set `canProject` to `false` for resources other than triggering one.
      */
@@ -212,7 +272,7 @@ private boolean loadCurrentObject(OperationResult result)
         LOGGER.trace("Trying to load current object");
 
         if (projectionContext.isAdd() && !projectionContext.isCompleted()) {
-            LOGGER.trace("No need to load old object, there is none");
+            LOGGER.trace("No need to try to load old object, there is none");
             projectionContext.setExists(false);
             projectionContext.recompute();
             projectionObject = asObjectable(projectionContext.getObjectNew());
@@ -352,39 +412,6 @@ private void checkLoadedShadowConsistency(PrismObject<ShadowType> object) {
         }
     }
 
-    /**
-     * Do we need to reload already-loaded object?
-     *
-     * TODO reconsider this algorithm
-     */
-    private boolean needToReload() {
-        if (projectionContext.isDoReconciliation() && !projectionContext.isFullShadow()) {
-            LOGGER.trace("Will reload, because doing reconciliation (and do not have full shadow)");
-            return true;
-        }
-
-        // This is kind of brutal. But effective. We are reloading all higher-order dependencies
-        // before they are processed. This makes sure we have fresh state when they are re-computed.
-        // Because higher-order dependencies may have more than one projection context and the
-        // changes applied to one of them are not automatically reflected on on other. therefore we need to reload.
-        if (projectionContext.getOrder() == 0) {
-            LOGGER.trace("Not doing reconciliation; and context is NOT of higher-order -> no need to reload");
-            return false;
-        }
-
-        int executionWave = context.getExecutionWave();
-        int projCtxWave = projectionContext.getWave();
-        if (executionWave == projCtxWave - 1) {
-            LOGGER.trace("Reloading higher-order context because its wave has come (exec wave = {}, projection wave = {})",
-                    executionWave, projCtxWave);
-            return true;
-        } else {
-            LOGGER.trace("Not reloading higher-order context because its wave has not come (exec wave = {}, projection wave = {})",
-                    executionWave, projCtxWave);
-            return false;
-        }
-    }
-
     private Collection<SelectorOptions<GetOperationOptions>> createProjectionLoadingOptions() {
         GetOperationOptionsBuilder builder = beans.schemaService.getOperationOptionsBuilder()
                 //.readOnly() [not yet]