Merge branch 'master' of github.com:Evolveum/midpoint

dist/javadoc/pom.xml

@@ -24,11 +24,11 @@
     <properties>
         <!-- see notes in main midPoint POM -->
         <!--  MID-5666: For some reason deploying of javadoc was disabled -->
-        <!--  
+        <!--
         <maven.install.skip>true</maven.install.skip>
         <maven.deploy.skip>true</maven.deploy.skip>
-        <maven.test.skip>true</maven.test.skip>
         -->
+        <maven.test.skip>true</maven.test.skip>
     </properties>
 
     <dependencies>

gui/admin-gui/src/main/java/com/evolveum/midpoint/web/page/admin/reports/component/SearchFilterConfigurationPanel.java

@@ -122,6 +122,7 @@ private void initLayout() {
 
         WebMarkupContainer textFieldContainer = new WebMarkupContainer(ID_TEXT_DNS_CONTAINER);
         textFieldContainer.setOutputMarkupId(true);
+        textFieldContainer.add(new VisibleBehaviour(() -> FiledType.QUERY.equals(fieldType)));
         textFieldContainer.add(AttributeAppender.append("class", classGridModel));
         textFieldContainer.add(AttributeAppender.append("style", (IModel<?>) () -> {
             if (FiledType.QUERY.equals(fieldType)) {

gui/admin-gui/src/main/java/com/evolveum/midpoint/web/util/SchrodingerComponentInitListener.java

@@ -18,6 +18,7 @@
 import org.apache.wicket.Page;
 import org.apache.wicket.application.IComponentInitializationListener;
 import org.apache.wicket.behavior.Behavior;
+import org.apache.wicket.markup.html.image.Image;
 import org.apache.wicket.model.PropertyModel;
 import org.apache.wicket.model.StringResourceModel;
 import org.apache.wicket.request.Response;
@@ -54,11 +55,7 @@ public void onInitialize(Component component) {
     }
 
     private void handleId(Component component) {
-        if (component instanceof CaptchaPanel) {
-            writeDataAttribute(component, ATTR_ID, component.getId() + ((CaptchaPanel) component).getRandomText());
-        } else {
-            writeDataAttribute(component, ATTR_ID, component.getId());
-        }
+        writeDataAttribute(component, ATTR_ID, component.getId());
     }
 
     private void writeDataAttribute(Component component, String key, String value) {

infra/schema/src/main/java/com/evolveum/midpoint/schema/GetOperationOptionsBuilder.java

@@ -52,6 +52,8 @@ public interface GetOperationOptionsBuilder {
     GetOperationOptionsBuilder staleness(Long value);
     GetOperationOptionsBuilder forceRefresh();
     GetOperationOptionsBuilder forceRefresh(Boolean value);
+    GetOperationOptionsBuilder forceRetry();
+    GetOperationOptionsBuilder forceRetry(Boolean value);
     GetOperationOptionsBuilder distinct();
     GetOperationOptionsBuilder distinct(Boolean value);
     GetOperationOptionsBuilder attachDiagData();

infra/schema/src/main/java/com/evolveum/midpoint/schema/GetOperationOptionsBuilderImpl.java

@@ -201,6 +201,16 @@ public GetOperationOptionsBuilder forceRefresh(Boolean value) {
         return forPaths(opts -> opts.setForceRefresh(value));
     }
 
+    @Override
+    public GetOperationOptionsBuilder forceRetry() {
+        return forceRetry(true);
+    }
+
+    @Override
+    public GetOperationOptionsBuilder forceRetry(Boolean value) {
+        return forPaths(opts -> opts.setForceRetry(value));
+    }
+
     @Override
     public GetOperationOptionsBuilder distinct() {
         return distinct(true);

model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/lens/projector/ContextLoader.java

@@ -92,6 +92,7 @@ public class ContextLoader implements ProjectorProcessor {
     @Autowired private SecurityHelper securityHelper;
     @Autowired private ClockworkMedic medic;
     @Autowired private RelationRegistry relationRegistry;
+    @Autowired private SchemaService schemaService;
 
     private static final Trace LOGGER = TraceManager.getTrace(ContextLoader.class);
 
@@ -665,10 +666,7 @@ private <F extends FocusType> void loadLinkRefsFromFocus(LensContext<F> context,
             if (!relationRegistry.isMember(linkRefVal.getRelation())) {
                 LOGGER.trace("Inactive linkRef: will only refresh it, no processing. Relation={}, ref={}",
                         linkRefVal.getRelation(), linkRefVal);
-                // We do this just to restore old behavior that ensures that linked shadows are quick-refreshed,
-                // deleting e.g. expired pending operations (see TestMultiResource.test429).
-                var options = createGetOptions(context);
-                refreshLinkedShadow(oid, options, task, result);
+                refreshInactiveLinkedShadow(oid, context, task, result);
                 continue;
             }
             if (StringUtils.isBlank(oid)) {
@@ -686,7 +684,7 @@ private <F extends FocusType> void loadLinkRefsFromFocus(LensContext<F> context,
             //noinspection unchecked
             PrismObject<ShadowType> shadowFromLink = linkRefVal.getObject();
             if (shadowFromLink == null) {
-                var options = createGetOptions(context);
+                var options = createStandardGetOptions();
                 LOGGER.trace("Loading shadow {} from linkRef, options={}", oid, options);
                 try {
                     shadow = provisioningService.getObject(ShadowType.class, oid, options, task, result);
@@ -726,21 +724,35 @@ private <F extends FocusType> void loadLinkRefsFromFocus(LensContext<F> context,
         }
     }
 
-    private <F extends FocusType> Collection<SelectorOptions<GetOperationOptions>> createGetOptions(LensContext<F> context) {
-        GetOperationOptions rootOpts;
+    private <F extends FocusType> Collection<SelectorOptions<GetOperationOptions>> createStandardGetOptions() {
+        // Using NO_FETCH so we avoid reading in a full account. This is more efficient as we don't need full account here.
+        // We need to fetch from provisioning and not repository so the correct definition will be set.
+        return schemaService.getOperationOptionsBuilder()
+                .noFetch()
+                .pointInTime(PointInTimeType.FUTURE)
+                .build();
+    }
+
+    /**
+     * We do this just to restore old behavior that ensures that linked shadows are quick-refreshed,
+     * deleting e.g. expired pending operations (see TestMultiResource.test429).
+     */
+    private void refreshInactiveLinkedShadow(String oid, LensContext<?> context, Task task, OperationResult result) {
+        Collection<SelectorOptions<GetOperationOptions>> options;
         if (context.isDoReconciliationForAllProjections()) {
-            rootOpts = GetOperationOptions.createForceRetry();
+            // Ensures an attempt to complete any pending operations.
+            // TODO Shouldn't we include FUTURE option as well? E.g. to avoid failing on not-yet-created accounts?
+            //  (Fortunately, we ignore any exceptions but anyway: FUTURE is used in other cases in this class.)
+            options = schemaService.getOperationOptionsBuilder()
+                    .forceRetry()
+                    .build();
         } else {
-            // Using NO_FETCH so we avoid reading in a full account. This is more efficient as we don't need full account here.
-            // We need to fetch from provisioning and not repository so the correct definition will be set.
-            rootOpts = GetOperationOptions.createNoFetch();
-            rootOpts.setPointInTimeType(PointInTimeType.FUTURE);
+            // This ensures only minimal processing, e.g. the quick shadow refresh is done.
+            options = schemaService.getOperationOptionsBuilder()
+                    .noFetch()
+                    .pointInTime(PointInTimeType.FUTURE)
+                    .build();
         }
-        return SelectorOptions.createCollection(rootOpts);
-    }
-
-    private void refreshLinkedShadow(String oid, Collection<SelectorOptions<GetOperationOptions>> options, Task task,
-            OperationResult result) {
         try {
             provisioningService.getObject(ShadowType.class, oid, options, task, result);
         } catch (Exception e) {
@@ -1262,23 +1274,14 @@ private <F extends ObjectType> void finishLoadOfProjectionContext(LensContext<F>
                                 "Projection "+projContext.getHumanReadableName()+" with null OID, no representation and no resource OID in account sync context "+projContext);
                     }
                 } else {
-                    GetOperationOptions rootOptions = GetOperationOptions.createPointInTimeType(PointInTimeType.FUTURE);
-                    if (projContext.isDoReconciliation()) {
-                        rootOptions.setForceRefresh(true);
-                        if (SchemaConstants.CHANNEL_DISCOVERY_URI.equals(context.getChannel())) {
-                            // Avoid discovery loops
-                            rootOptions.setDoNotDiscovery(true);
-                        }
-                    } else {
-                        rootOptions.setNoFetch(true);
-                    }
-                    rootOptions.setAllowNotFound(true);
-                    Collection<SelectorOptions<GetOperationOptions>> options = SelectorOptions.createCollection(rootOptions);
-                    LOGGER.trace("Loading shadow {} for projection {}, options={}", projectionObjectOid, projContext.getHumanReadableName(), options);
+                    Collection<SelectorOptions<GetOperationOptions>> options = createProjectionLoadingOptions(projContext);
+                    LOGGER.trace("Loading shadow {} for projection {}, options={}", projectionObjectOid,
+                            projContext.getHumanReadableName(), options);
 
                     try {
                         PrismObject<ShadowType> objectOld = provisioningService.getObject(
                                 projContext.getObjectTypeClass(), projectionObjectOid, options, task, result);
+                        GetOperationOptions rootOptions = SelectorOptions.findRootOptions(options);
                         if (!GetOperationOptions.isNoFetch(rootOptions) && !GetOperationOptions.isRaw(rootOptions)) {
                             LOGGER.trace("Full shadow loaded for {}:\n{}", projContext.getHumanReadableName(), objectOld.debugDumpLazily(1));
                         }
@@ -1430,6 +1433,35 @@ private <F extends ObjectType> void finishLoadOfProjectionContext(LensContext<F>
         setPrimaryDeltaOldValue(projContext);
     }
 
+    private Collection<SelectorOptions<GetOperationOptions>> createProjectionLoadingOptions(LensProjectionContext projContext) {
+        GetOperationOptionsBuilder builder = schemaService.getOperationOptionsBuilder()
+                .pointInTime(PointInTimeType.FUTURE)
+                .allowNotFound();
+
+        LensContext<?> context = projContext.getLensContext();
+
+        // Most probably reconciliation for all projections implies reconciliation for projContext
+        // but we include both conditions just to be sure.
+        if (projContext.isDoReconciliation() || context.isDoReconciliationForAllProjections()) {
+            builder = builder.forceRefresh();
+
+            // We force operation retry "in hard way" only if we do full-scale reconciliation AND we are starting the clockwork.
+            // This is to avoid useless repetition of retries (pushing attempt number quickly too high).
+            if (context.isDoReconciliationForAllProjections() && context.getProjectionWave() == 0) {
+                builder = builder.forceRetry();
+            }
+
+            if (SchemaConstants.CHANNEL_DISCOVERY_URI.equals(context.getChannel())) {
+                // Avoid discovery loops
+                builder = builder.doNotDiscovery();
+            }
+        } else {
+            builder = builder.noFetch();
+        }
+
+        return builder.build();
+    }
+
     private <F extends ObjectType> boolean needToReload(LensContext<F> context,
             LensProjectionContext projContext) {
         ResourceShadowDiscriminator discr = projContext.getResourceShadowDiscriminator();

model/model-intest/src/test/java/com/evolveum/midpoint/model/intest/orgstruct/TestOrgStruct.java

@@ -550,10 +550,8 @@ public void test232JackDestroyRefsAndRecompute() throws Exception {
 
         assertRefs23x();
 
-        // Why so many operations? But this is a very special case. As long as we do not see significant
-        // increase of operation count in normal scenarios we are quite OK.
-        assertCounterIncrement(InternalCounters.SHADOW_FETCH_OPERATION_COUNT, 4);
-        assertCounterIncrement(InternalCounters.CONNECTOR_OPERATION_COUNT, 8);
+        assertCounterIncrement(InternalCounters.SHADOW_FETCH_OPERATION_COUNT, 2);
+        assertCounterIncrement(InternalCounters.CONNECTOR_OPERATION_COUNT, 5);
     }
 
     /**

provisioning/provisioning-api/src/main/java/com/evolveum/midpoint/provisioning/api/ProvisioningOperationOptions.java

@@ -272,6 +272,7 @@ public void shortDump(StringBuilder sb) {
         if (runAsAccountOid != null) {
             sb.append("runAsAccountOid=").append(runAsAccountOid).append(",");
         }
+        appendFlag(sb, "forceRetry", forceRetry);
         if (isEmpty()) {
             sb.append("(empty)");
         } else {

provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/ProvisioningOperationState.java

@@ -10,10 +10,10 @@
 import java.util.List;
 
 import com.evolveum.midpoint.prism.PrismObject;
-import com.evolveum.midpoint.provisioning.api.ShadowDeathListener;
 import com.evolveum.midpoint.schema.result.AsynchronousOperationResult;
 import com.evolveum.midpoint.schema.result.OperationResult;
 import com.evolveum.midpoint.schema.result.OperationResultStatus;
+import com.evolveum.midpoint.schema.util.ShadowUtil;
 import com.evolveum.midpoint.util.ShortDumpable;
 import com.evolveum.midpoint.xml.ns._public.common.common_3.OperationResultStatusType;
 import com.evolveum.midpoint.xml.ns._public.common.common_3.PendingOperationExecutionStatusType;
@@ -221,4 +221,11 @@ public static <A extends AsynchronousOperationResult> ProvisioningOperationState
         return opState;
     }
 
+    public boolean objectExists() {
+        if (repoShadow != null) {
+            return ShadowUtil.isExists(repoShadow);
+        } else {
+            return false; // shouldn't occur
+        }
+    }
 }

provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/shadows/RefreshHelper.java

@@ -468,19 +468,26 @@ private void retryOperation(ProvisioningContext ctx, ObjectDelta<ShadowType> pen
             addHelper.addShadowAttempt(ctx, resourceObjectToAdd, scripts,
                     (ProvisioningOperationState<AsynchronousOperationReturnValue<PrismObject<ShadowType>>>) opState,
                     options, task, result);
-            opState.setRepoShadow(resourceObjectToAdd);
         }
 
         if (pendingDelta.isModify()) {
-            modifyHelper.modifyShadowAttempt(ctx, pendingDelta.getModifications(), scripts, options,
-                    (ProvisioningOperationState<AsynchronousOperationReturnValue<Collection<PropertyDelta<PrismPropertyValue>>>>) opState,
-                    true, task, result);
+            if (opState.objectExists()) {
+                modifyHelper.modifyShadowAttempt(ctx, pendingDelta.getModifications(), scripts, options,
+                        (ProvisioningOperationState<AsynchronousOperationReturnValue<Collection<PropertyDelta<PrismPropertyValue>>>>) opState,
+                        true, task, result);
+            } else {
+                result.recordFatalError("Object does not exist on the resource yet, modification attempt was skipped");
+            }
         }
 
         if (pendingDelta.isDelete()) {
-            deleteHelper.deleteShadowAttempt(ctx, options, scripts,
-                    (ProvisioningOperationState<AsynchronousOperationResult>) opState,
-                    task, result);
+            if (opState.objectExists()) {
+                deleteHelper.deleteShadowAttempt(ctx, options, scripts,
+                        (ProvisioningOperationState<AsynchronousOperationResult>) opState,
+                        task, result);
+            } else {
+                result.recordFatalError("Object does not exist on the resource yet, deletion attempt was skipped");
+            }
         }
     }
 

provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/shadows/manager/PendingOperationsHelper.java

@@ -108,7 +108,7 @@ void collectPendingOperationUpdates(Collection<ItemDelta<?, ?>> shadowChanges,
                 // This must be a new operation
                 ContainerDelta<PendingOperationType> cdelta = prismContext.deltaFactory().container().create(
                         ShadowType.F_PENDING_OPERATION, containerDefinition);
-                cdelta.addValuesToAdd(pendingOperation.asPrismContainerValue());
+                cdelta.addValuesToAdd(pendingOperation.asPrismContainerValue().clone());
                 shadowChanges.add(cdelta);
             } else {
                 ItemPath containerPath = pendingOperation.asPrismContainerValue().getPath();

provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/shadows/manager/ShadowUpdater.java

@@ -99,7 +99,7 @@ class ShadowUpdater {
     /**
      * Record results of ADD operation to the shadow.
      */
-    public void recordAddResult(ProvisioningContext ctx, PrismObject<ShadowType> shadowToAdd,
+    void recordAddResult(ProvisioningContext ctx, PrismObject<ShadowType> shadowToAdd,
             ProvisioningOperationState<AsynchronousOperationReturnValue<PrismObject<ShadowType>>> opState,
             OperationResult result)
             throws SchemaException, ConfigurationException, ObjectNotFoundException, CommunicationException,
@@ -168,12 +168,11 @@ private void recordAddResultNewShadow(ProvisioningContext ctx, PrismObject<Shado
         result.recordSuccess();
     }
 
-    private void recordAddResultExistingShadow(
-            ProvisioningContext ctx,
-            PrismObject<ShadowType> shadowToAdd,
+    private void recordAddResultExistingShadow(ProvisioningContext ctx, PrismObject<ShadowType> shadowToAdd,
             ProvisioningOperationState<AsynchronousOperationReturnValue<PrismObject<ShadowType>>> opState,
             OperationResult result)
-            throws SchemaException, ConfigurationException, ObjectNotFoundException, CommunicationException, ObjectAlreadyExistsException, ExpressionEvaluationException {
+            throws SchemaException, ConfigurationException, ObjectNotFoundException, CommunicationException,
+            ExpressionEvaluationException {
 
         final PrismObject<ShadowType> resourceShadow;
         if (opState.wasStarted() && opState.getAsyncResult().getReturnValue() != null) {
@@ -431,6 +430,9 @@ private void executeShadowModification(ProvisioningContext ctx, PrismObject<Shad
                 if (wasMarkedDead(shadow, modifications)) {
                     eventDispatcher.notify(ShadowDeathEvent.dead(shadow.getOid()), ctx.getTask(), result);
                 }
+                // This is important e.g. to update opState.repoShadow content in case of ADD operation success
+                // - to pass newly-generated primary identifier to other parts of the code.
+                ItemDeltaCollectionsUtil.applyTo(modifications, shadow);
                 LOGGER.trace("Shadow changes processed successfully.");
             } catch (ObjectAlreadyExistsException ex) {
                 throw new SystemException(ex);