Merge remote-tracking branch 'refs/remotes/origin/master'

infra/schema/src/main/java/com/evolveum/midpoint/schema/constants/SchemaConstants.java

@@ -314,6 +314,8 @@ public abstract class SchemaConstants {
     public static final ItemName MODEL_EXTENSION_OBJECTCLASS = new ItemName(NS_MODEL_EXTENSION, "objectclass");
     public static final ItemName MODEL_EXTENSION_LAST_SCAN_TIMESTAMP_PROPERTY_NAME = new ItemName(
             NS_MODEL_EXTENSION, "lastScanTimestamp");
+    public static final ItemName MODEL_EXTENSION_LAST_RECONCILIATION_START_TIMESTAMP_PROPERTY_NAME = new ItemName(
+            NS_MODEL_EXTENSION, "lastReconciliationStartTimestamp");
     public static final ItemName MODEL_EXTENSION_PROFILING_INTERVAL = new ItemName(NS_MODEL_EXTENSION, "profilingInterval");
     public static final ItemName MODEL_EXTENSION_TRACING_INTERVAL = new ItemName(NS_MODEL_EXTENSION, "tracingInterval");
     public static final ItemName MODEL_EXTENSION_TRACING_PROFILE = new ItemName(NS_MODEL_EXTENSION, "tracingProfile");

infra/schema/src/main/java/com/evolveum/midpoint/schema/util/ResourceTypeUtil.java

@@ -722,15 +722,19 @@ public static boolean isRefreshOnRead(ResourceType resource) {
         if (consistency == null) {
             return false;
         }
+
         Boolean refreshOnRead = consistency.isRefreshOnRead();
-        if (refreshOnRead == null) {
-            Boolean reshreshOnRead = consistency.isReshreshOnRead();
-            if (reshreshOnRead == null) {
-                return false;
-            }
-            return false;
+        if (refreshOnRead != null) {
+            return refreshOnRead;
         }
-        return refreshOnRead;
+
+        // legacy way (misspelled property)
+        Boolean reshreshOnRead = consistency.isReshreshOnRead();
+        if (reshreshOnRead != null) {
+            return reshreshOnRead;
+        }
+
+        return false;
     }
 
     public static ErrorSelectorType getConnectorErrorCriticality(ResourceType resourceType) {

infra/schema/src/main/resources/xml/ns/public/model/extension-3.xsd

@@ -64,6 +64,7 @@
             <xsd:element ref="tns:updateLiveSyncTokenInDryRun" minOccurs="0"/>
             <xsd:element ref="tns:finishOperationsOnly" minOccurs="0"/>
             <xsd:element ref="tns:lastScanTimestamp" minOccurs="0"/>
+            <xsd:element ref="tns:lastReconciliationStartTimestamp" minOccurs="0"/>
             <xsd:element ref="tns:objectQuery" minOccurs="0"/>
             <xsd:element ref="tns:searchOptions" minOccurs="0"/>
             <xsd:element ref="tns:useRepositoryDirectly" minOccurs="0"/>
@@ -231,6 +232,28 @@
         </xsd:annotation>
     </xsd:element>
 
+    <xsd:element name="lastReconciliationStartTimestamp" type="xsd:dateTime">
+        <xsd:annotation>
+            <xsd:documentation>
+                When the resource reconciliation last started? This information is used to determine
+                the threshold to consider when searching for untouched shadows in 3rd part of a reconciliation
+                (shadow reconciliation). Should be set on the root of the task tree.
+
+                The value is set during the first part of the reconciliation.
+
+                EXPERIMENTAL. Probably will be replaced by something more serious during task development for midPoint 4.4.
+            </xsd:documentation>
+            <xsd:appinfo>
+                <a:displayName>TaskExtension.lastReconciliationStartTimestamp</a:displayName>
+                <a:minOccurs>0</a:minOccurs>
+                <a:maxOccurs>1</a:maxOccurs>
+                <a:since>4.3</a:since>
+                <a:operational>true</a:operational>
+                <a:experimental>true</a:experimental>
+            </xsd:appinfo>
+        </xsd:annotation>
+    </xsd:element>
+
     <xsd:element name="objectQuery" type="q:QueryType">
         <xsd:annotation>
             <xsd:documentation>

model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/sync/tasks/recon/ReconciliationTaskFirstPartExecution.java

@@ -12,17 +12,19 @@
 import com.evolveum.midpoint.model.impl.util.ModelImplUtils;
 import com.evolveum.midpoint.prism.PrismObject;
 import com.evolveum.midpoint.prism.query.ObjectQuery;
+import com.evolveum.midpoint.prism.xml.XmlTypeConverter;
 import com.evolveum.midpoint.provisioning.api.ProvisioningOperationOptions;
 import com.evolveum.midpoint.repo.api.PreconditionViolationException;
 import com.evolveum.midpoint.repo.cache.RepositoryCache;
 import com.evolveum.midpoint.repo.common.task.AbstractSearchIterativeItemProcessor;
 import com.evolveum.midpoint.repo.common.task.HandledObjectType;
 import com.evolveum.midpoint.repo.common.task.ItemProcessingRequest;
 import com.evolveum.midpoint.repo.common.task.ItemProcessorClass;
+import com.evolveum.midpoint.schema.constants.SchemaConstants;
 import com.evolveum.midpoint.schema.result.OperationResult;
 import com.evolveum.midpoint.task.api.RunningTask;
-import com.evolveum.midpoint.util.exception.CommonException;
-import com.evolveum.midpoint.util.exception.SchemaException;
+import com.evolveum.midpoint.task.api.Task;
+import com.evolveum.midpoint.util.exception.*;
 import com.evolveum.midpoint.xml.ns._public.common.common_3.ShadowType;
 
 /**
@@ -61,6 +63,25 @@ protected ObjectQuery createQuery(OperationResult opResult) throws SchemaExcepti
     protected void finish(OperationResult opResult) throws SchemaException {
         super.finish(opResult);
         taskExecution.reconResult.setUnOpsCount(bucketStatistics.getItemsProcessed());
+        setLastReconciliationStartTimestamp(opResult);
+    }
+
+    /**
+     * Sets "lastReconciliationStartTimestamp" property in the root task. This is needed to establish a threshold
+     * for selection of shadows untouched in the resource reconciliation (i.e. the 2nd part of the whole process).
+     *
+     * It does not matter how many times this method is called during first part execution (although in almost any situation
+     * it is called exactly once). We are interested in the latest value.
+     */
+    private void setLastReconciliationStartTimestamp(OperationResult result) throws SchemaException {
+        Task rootTask = getRootTask(result);
+        rootTask.setExtensionPropertyValue(SchemaConstants.MODEL_EXTENSION_LAST_RECONCILIATION_START_TIMESTAMP_PROPERTY_NAME,
+                XmlTypeConverter.createXMLGregorianCalendar());
+        try {
+            rootTask.flushPendingModifications(result);
+        } catch (ObjectNotFoundException | ObjectAlreadyExistsException e) {
+            throw new SystemException("Couldn't set last reconciliation start timestamp in root task " + rootTask, e);
+        }
     }
 
     protected static class ItemProcessor

model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/sync/tasks/recon/ReconciliationTaskThirdPartExecution.java

@@ -72,7 +72,7 @@ protected ObjectQuery createQuery(OperationResult opResult) throws SchemaExcepti
             CommunicationException, ConfigurationException, ExpressionEvaluationException, SecurityViolationException {
         ObjectQuery initialQuery = getPrismContext().queryFor(ShadowType.class)
                 .block()
-                    .item(ShadowType.F_FULL_SYNCHRONIZATION_TIMESTAMP).le(getReconciliationStartTimestamp())
+                    .item(ShadowType.F_FULL_SYNCHRONIZATION_TIMESTAMP).le(getReconciliationStartTimestamp(opResult))
                     .or().item(ShadowType.F_FULL_SYNCHRONIZATION_TIMESTAMP).isNull()
                 .endBlock()
                 .and().item(ShadowType.F_RESOURCE_REF).ref(taskExecution.getResourceOid())
@@ -81,11 +81,19 @@ protected ObjectQuery createQuery(OperationResult opResult) throws SchemaExcepti
         return taskExecution.createShadowQuery(initialQuery, opResult);
     }
 
-    private XMLGregorianCalendar getReconciliationStartTimestamp() {
-        // TODO TODO TODO
-        //  We should provide start timestamp of the start of the second stage.
-        //  It could be present in a separate task (!)
-        return taskExecution.startTimestamp;
+    private XMLGregorianCalendar getReconciliationStartTimestamp(OperationResult result) throws SchemaException {
+        Task rootTask = getRootTask(result);
+        XMLGregorianCalendar lastReconStart = rootTask.getExtensionPropertyRealValue(
+                SchemaConstants.MODEL_EXTENSION_LAST_RECONCILIATION_START_TIMESTAMP_PROPERTY_NAME);
+        if (lastReconStart != null) {
+            logger.trace("Last reconciliation start time: {}, determined from the extension of {}", lastReconStart, rootTask);
+            return lastReconStart;
+        } else {
+            XMLGregorianCalendar implicitLastReconStart = taskExecution.startTimestamp;
+            logger.trace("Last reconciliation start time: {}, determined as start timestamp of the respective part in {}",
+                    implicitLastReconStart, localCoordinatorTask);
+            return implicitLastReconStart;
+        }
     }
 
     @Override

model/model-intest/src/test/java/com/evolveum/midpoint/model/intest/sync/AbstractSynchronizationStoryTest.java

@@ -94,7 +94,7 @@ protected boolean isReconciliation() {
     }
 
     @Test
-    public void test100ImportLiveSyncTaskDummyGreen() throws Exception {
+    public void test100ImportSyncTaskDummyGreen() throws Exception {
         when();
         importSyncTask(getDummyResourceObject(RESOURCE_DUMMY_GREEN_NAME));
 
@@ -157,6 +157,12 @@ public void test110AddDummyGreenAccountMancomb() throws Exception {
             assertThat(provisioningStatistics.getEntry().get(0).getResourceRef().getOid()).isEqualTo(RESOURCE_DUMMY_GREEN_OID);
             assertThat(getOrig(provisioningStatistics.getEntry().get(0).getResourceRef().getTargetName())).isEqualTo("Dummy Resource Green");
             assertThat(provisioningStatistics.getEntry().get(0).getOperation()).isNotEmpty(); // search and sometimes get
+
+            Integer itemsProcessed = TaskOperationStatsUtil.getItemsProcessed(stats);
+
+            // MID-6930: We should process exactly 1 item even for partitioned reconciliation:
+            // mancomb must not be processed in the 3rd part!
+            assertThat(itemsProcessed).as("items processed").isEqualTo(1);
         }
 
         // notifications
@@ -168,7 +174,7 @@ public void test110AddDummyGreenAccountMancomb() throws Exception {
     protected abstract String getExpectedChannel();
 
     @Test
-    public void test200ImportLiveSyncTaskDummyBlue() throws Exception {
+    public void test200ImportSyncTaskDummyBlue() throws Exception {
         when();
         importSyncTask(getDummyResourceObject(RESOURCE_DUMMY_BLUE_NAME));
 
@@ -325,7 +331,7 @@ public void test315AddDummyBlueAccountMancomb() throws Exception {
      * the default dummy account directly. Just make sure that it does not do anything bad.
      */
     @Test
-    public void test350ImportLiveSyncTaskDummyDefault() throws Exception {
+    public void test350ImportSyncTaskDummyDefault() throws Exception {
         when();
         importSyncTask(getDummyResourceObject());
 

model/model-intest/src/test/java/com/evolveum/midpoint/model/intest/sync/TestReconTaskMultiNode.java

@@ -18,7 +18,7 @@
  * The same as TestReconTaskPartitioned but the second partition (resource reconciliation) is executed in a set of worker tasks.
  * (Currently there is only a single bucket, but multiple bucket processing will be implemented shortly.)
  *
- * Cannot be run under H2 because of too much contention.
+ * Shouldn't be run under H2 because of too much contention.
  * Also, it takes a little longer than standard TestReconTask because of the overhead.
  */
 @ContextConfiguration(locations = {"classpath:ctx-model-intest-test-main.xml"})

model/model-intest/src/test/java/com/evolveum/midpoint/model/intest/sync/TestReconTaskPartitioned.java

@@ -21,7 +21,7 @@
  * The same as TestReconTask but this one uses partitioned reconciliation task handler.
  * I.e. each reconciliation task is divided into three subtasks (for stage 1, 2, 3).
  *
- * Cannot be run under H2 because of too much contention.
+ * Shouldn't be run under H2 because of too much contention.
  * Also, it takes a little longer than standard TestReconTask because of the overhead.
  */
 @ContextConfiguration(locations = {"classpath:ctx-model-intest-test-main.xml"})

pom.xml

@@ -903,7 +903,7 @@
             <dependency>
                 <groupId>com.evolveum.polygon</groupId>
                 <artifactId>connector-ldap</artifactId>
-                <version>3.2-M4</version>
+                <version>3.2-M5</version>
                 <exclusions>
                     <!-- Needed otherwise the JDK14 SLF4J binding can override the midpoint's logback binding -->
                     <exclusion>

provisioning/provisioning-impl/src/test/java/com/evolveum/midpoint/provisioning/impl/opendj/TestOpenDj.java

@@ -1759,7 +1759,7 @@ public void test206SearchObjectsComplexFilterStartsWith() throws Exception {
      *
      * MID-6898
      */
-    @Test(enabled = false)
+    @Test
     public void test210SearchObjectsFromAnotherObjectClass() throws Exception {
         Task task = getTestTask();
         OperationResult result = task.getResult();

provisioning/provisioning-impl/src/test/resources/opendj/resource-opendj-dumber.xml

@@ -189,6 +189,26 @@ somehow dumber: no shortcut in associations, expclicit duplicity checks, etc.
             </attribute>
         </objectType>
 
+        <objectType>
+            <!-- Yet another entitlement. Same objectclass, but no baseContext. Testing MID-6898. -->
+            <kind>entitlement</kind>
+            <intent>unlimitedGroup</intent>
+            <objectClass>ri:groupOfUniqueNames</objectClass>
+            <attribute>
+                <ref>ri:entryUUID</ref>
+                <matchingRule>mr:stringIgnoreCase</matchingRule>
+            </attribute>
+            <attribute>
+                <ref>ri:dn</ref>
+                <matchingRule>mr:distinguishedName</matchingRule>
+            </attribute>
+            <attribute>
+                <ref>ri:uniqueMember</ref>
+                <matchingRule>mr:distinguishedName</matchingRule>
+                <fetchStrategy>minimal</fetchStrategy>
+            </attribute>
+        </objectType>
+
         <objectType>
             <!-- OrganizationUnit with base context pointing to organizationUnit.
                  Make sure this won't end up in an endless loop. -->

provisioning/provisioning-impl/src/test/resources/opendj/resource-opendj-incomplete-password.xml

@@ -183,6 +183,26 @@
             </attribute>
         </objectType>
 
+        <objectType>
+            <!-- Yet another entitlement. Same objectclass, but no baseContext. Testing MID-6898. -->
+            <kind>entitlement</kind>
+            <intent>unlimitedGroup</intent>
+            <objectClass>ri:groupOfUniqueNames</objectClass>
+            <attribute>
+                <ref>ri:entryUUID</ref>
+                <matchingRule>mr:stringIgnoreCase</matchingRule>
+            </attribute>
+            <attribute>
+                <ref>ri:dn</ref>
+                <matchingRule>mr:distinguishedName</matchingRule>
+            </attribute>
+            <attribute>
+                <ref>ri:uniqueMember</ref>
+                <matchingRule>mr:distinguishedName</matchingRule>
+                <fetchStrategy>minimal</fetchStrategy>
+            </attribute>
+        </objectType>
+
         <objectType>
             <!-- OrganizationUnit with base context pointing to organizationUnit.
                  Make sure this won't end up in an endless loop. -->

provisioning/provisioning-impl/src/test/resources/opendj/resource-opendj-readable-password.xml

@@ -186,6 +186,26 @@
             </attribute>
         </objectType>
 
+        <objectType>
+            <!-- Yet another entitlement. Same objectclass, but no baseContext. Testing MID-6898. -->
+            <kind>entitlement</kind>
+            <intent>unlimitedGroup</intent>
+            <objectClass>ri:groupOfUniqueNames</objectClass>
+            <attribute>
+                <ref>ri:entryUUID</ref>
+                <matchingRule>mr:stringIgnoreCase</matchingRule>
+            </attribute>
+            <attribute>
+                <ref>ri:dn</ref>
+                <matchingRule>mr:distinguishedName</matchingRule>
+            </attribute>
+            <attribute>
+                <ref>ri:uniqueMember</ref>
+                <matchingRule>mr:distinguishedName</matchingRule>
+                <fetchStrategy>minimal</fetchStrategy>
+            </attribute>
+        </objectType>
+
         <objectType>
             <!-- OrganizationUnit with base context pointing to organizationUnit.
                  Make sure this won't end up in an endless loop. -->

provisioning/ucf-impl-connid/src/test/resources/connector-ldap.xml

@@ -14,7 +14,7 @@
     <name>ICF com.evolveum.polygon.connector.ldap.LdapConnector</name>
     <framework>http://midpoint.evolveum.com/xml/ns/public/connector/icf-1</framework>
     <connectorType>com.evolveum.polygon.connector.ldap.LdapConnector</connectorType>
-    <connectorVersion>3.2-M4</connectorVersion>
+    <connectorVersion>3.2-M5</connectorVersion>
     <connectorBundle>com.evolveum.polygon.connector-ldap</connectorBundle>
     <namespace>http://midpoint.evolveum.com/xml/ns/public/connector/icf-1/bundle/com.evolveum.polygon.connector-ldap/com.evolveum.polygon.connector.ldap.LdapConnector</namespace>
     <schema>

repo/repo-common/src/main/java/com/evolveum/midpoint/repo/common/task/AbstractIterativeTaskPartExecution.java

@@ -14,6 +14,7 @@
 import com.evolveum.midpoint.schema.util.task.TaskOperationStatsUtil;
 import com.evolveum.midpoint.schema.util.task.TaskPartPerformanceInformation;
 import com.evolveum.midpoint.schema.util.task.TaskProgressUtil;
+import com.evolveum.midpoint.task.api.Task;
 import com.evolveum.midpoint.util.annotation.Experimental;
 import com.evolveum.midpoint.xml.ns._public.common.common_3.*;
 
@@ -481,10 +482,24 @@ public boolean isSimulate() {
         return partDefinition != null && partDefinition.getStage() == ExecutionModeType.SIMULATE;
     }
 
-    public String getRootTaskOid() {
+    public @NotNull String getRootTaskOid() {
         return taskExecution.getRootTaskOid();
     }
 
+    protected @NotNull Task getRootTask(OperationResult result) throws SchemaException {
+        String rootTaskOid = getRootTaskOid();
+        if (localCoordinatorTask.getOid().equals(rootTaskOid)) {
+            return localCoordinatorTask;
+        } else {
+            try {
+                return taskHandler.taskManager.getTaskPlain(rootTaskOid, result);
+            } catch (ObjectNotFoundException e) {
+                // This is quite unexpected so it can be rethrown as SystemException
+                throw new SystemException("The root task was not found", e);
+            }
+        }
+    }
+
     public String getPartUri() {
         return partUri;
     }