Skip to content

Commit

Permalink
Improve Clockwork and Projector code a bit
Browse files Browse the repository at this point in the history 
The goal of this work is to make the structure of the processing
in Projector (and partially also in Clockwork) more obvious from
the source code.

1. Eliminated the wave loop from Projector.projectInternal;
moving it into Projector.projectAllWaves. (Note that the projector
wave loop is used only for previewing changes.)

2. AssignmentHolderProcessor was cleaned up significantly, factoring out
e.g. iteration functionality into IterationHelper.

3. Common entry code in processors (checking focus type and existence,
casting LensContext) was factored out to ClockworkMedic, using
experimental ProcessorExecution annotation.

4. Common post-processing in processors (integrity checking,
cross-component updating, diag logging) was moved to processors
themselves, simplifying the orchestration code.

5. PolicyRuleEnforcer.execute and policyRuleSuspendTaskExecutor.execute
calls were moved to more appropriate places.

6. FocusProcessor was renamed to FocusActivationProcessor.

A couple of other minor changes were done as well, with (presumably)
negligible impact on the functionality.
  • Loading branch information
@mederly
mederly committed Apr 23, 2020
1 parent 80a957c commit 2c98457
Show file tree
Hide file tree
Showing 43 changed files with 4,469 additions and 4,474 deletions.
2 changes: 1 addition & 1 deletion ...dmin-gui/src/main/java/com/evolveum/midpoint/web/component/progress/ProgressReporter.java
View file
Expand Up @@ -53,7 +53,7 @@ public class ProgressReporter implements ProgressListener {

private Map<String, String> nameCache = new HashMap<>();
private ProgressDto progress = new ProgressDto();
private boolean abortRequested;
private volatile boolean abortRequested;

// Operation result got from the asynchronous operation (null if async op not yet finished)
private OperationResult asyncOperationResult;
Expand Down
14 changes: 7 additions & 7 deletions infra/schema/src/main/java/com/evolveum/midpoint/schema/internals/InternalsConfig.java
View file
Expand Up @@ -54,7 +54,7 @@ public class InternalsConfig {
*/
private static TestingPaths testingPaths = null;

private static boolean detailedAuhotizationLog = false;
private static boolean detailedAuthorizationLog = false;

public static boolean isPrismMonitoring() {
return prismMonitoring;
Expand Down Expand Up @@ -120,12 +120,12 @@ public static void setTestingPaths(TestingPaths testingPaths) {
InternalsConfig.testingPaths = testingPaths;
}

public static boolean isDetailedAuhotizationLog() {
return detailedAuhotizationLog;
public static boolean isDetailedAuthorizationLog() {
return detailedAuthorizationLog;
}

public static void setDetailedAuhotizationLog(boolean detailedAuhotizationLog) {
InternalsConfig.detailedAuhotizationLog = detailedAuhotizationLog;
public static void setDetailedAuthorizationLog(boolean detailedAuthorizationLog) {
InternalsConfig.detailedAuthorizationLog = detailedAuthorizationLog;
}

public static boolean isAllowClearDataLogging() {
Expand Down Expand Up @@ -159,7 +159,7 @@ public static void set(Configuration internalsConfig) {
prismMonitoring = internalsConfig.getBoolean("prismMonitoring", prismMonitoring);
modelProfiling = internalsConfig.getBoolean("modelProfiling", modelProfiling);
// TODO: testingPaths
detailedAuhotizationLog = internalsConfig.getBoolean("detailedAuhotizationLog", detailedAuhotizationLog);
detailedAuthorizationLog = internalsConfig.getBoolean("detailedAuhotizationLog", detailedAuthorizationLog);

}

Expand All @@ -173,7 +173,7 @@ public static void reset() {
prismMonitoring = false;
modelProfiling = false;
testingPaths = null;
detailedAuhotizationLog = false;
detailedAuthorizationLog = false;
}

public static void setDevelopmentMode() {
Expand Down
2 changes: 1 addition & 1 deletion model/model-api/src/main/java/com/evolveum/midpoint/model/api/util/ClockworkInspector.java
View file
Expand Up @@ -13,7 +13,7 @@
import com.evolveum.midpoint.xml.ns._public.common.common_3.ObjectType;

/**
* Interface used to intercept the SyncContext as it passes through the computation.
* Interface used to intercept the ModelContext as it passes through the computation.
*
* It is mostly used in tests.
*
Expand Down
181 changes: 117 additions & 64 deletions model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/lens/Clockwork.java
View file
Expand Up @@ -16,6 +16,11 @@
import java.util.Collection;
import javax.xml.datatype.XMLGregorianCalendar;

import com.evolveum.midpoint.model.impl.lens.projector.Components;

import com.evolveum.midpoint.model.impl.lens.projector.policy.PolicyRuleEnforcer;
import com.evolveum.midpoint.prism.delta.ReferenceDelta;

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Component;
Expand Down Expand Up @@ -52,7 +57,6 @@
import com.evolveum.midpoint.schema.cache.CacheConfigurationManager;
import com.evolveum.midpoint.schema.cache.CacheType;
import com.evolveum.midpoint.schema.constants.SchemaConstants;
import com.evolveum.midpoint.schema.internals.InternalsConfig;
import com.evolveum.midpoint.schema.result.OperationResult;
import com.evolveum.midpoint.schema.result.OperationResultBuilder;
import com.evolveum.midpoint.schema.result.OperationResultStatus;
Expand Down Expand Up @@ -95,6 +99,7 @@ public class Clockwork {
@Autowired private Migrator migrator;
@Autowired private ClockworkMedic medic;
@Autowired private PolicyRuleScriptExecutor policyRuleScriptExecutor;
@Autowired private PolicyRuleEnforcer policyRuleEnforcer;
@Autowired private PolicyRuleSuspendTaskExecutor policyRuleSuspendTaskExecutor;
@Autowired private ClockworkAuthorizationHelper clockworkAuthorizationHelper;
@Autowired private CacheConfigurationManager cacheConfigurationManager;
Expand Down Expand Up @@ -124,9 +129,7 @@ public <F extends ObjectType> HookOperationMode run(LensContext<F> context, Task
}

LOGGER.trace("Running clockwork for context {}", context);
if (InternalsConfig.consistencyChecks) {
context.checkConsistence();
}
context.checkConsistenceIfNeeded();

int clicked = 0;
ClockworkConflictResolver.Context conflictResolutionContext = new ClockworkConflictResolver.Context();
Expand All @@ -141,6 +144,8 @@ public <F extends ObjectType> HookOperationMode run(LensContext<F> context, Task
//enterDefaultSearchExpressionEvaluatorCache();
provisioningService.enterConstraintsCheckerCache();

executeInitialChecks(context);

while (context.getState() != ModelState.FINAL) {

int maxClicks = getMaxClicks(result);
Expand Down Expand Up @@ -191,6 +196,46 @@ public <F extends ObjectType> HookOperationMode run(LensContext<F> context, Task
}
}

private <F extends ObjectType> void executeInitialChecks(LensContext<F> context) throws PolicyViolationException {
if (context.hasFocusOfType(AssignmentHolderType.class)) {
checkArchetypeRefDelta(context);
}
}

/**
* This check was originally present in AssignmentHolderProcessor. But it refers to focus primary delta only;
* so it's sufficient to execute it on clockwork entry (unless primary delta is manipulated e.g. in a scripting hook).
*/
private <F extends ObjectType> void checkArchetypeRefDelta(LensContext<F> context) throws PolicyViolationException {
ObjectDelta<F> focusPrimaryDelta = context.getFocusContext().getPrimaryDelta();
if (focusPrimaryDelta != null) {
ReferenceDelta archetypeRefDelta = focusPrimaryDelta.findReferenceModification(AssignmentHolderType.F_ARCHETYPE_REF);
if (archetypeRefDelta != null) {
// We want to allow this under special circumstances. E.g. we want be able to import user with archetypeRef.
// Otherwise we won't be able to export a user and re-import it again.
if (focusPrimaryDelta.isAdd()) {
String archetypeOidFromAssignments = LensUtil.determineExplicitArchetypeOidFromAssignments(focusPrimaryDelta.getObjectToAdd());
if (archetypeOidFromAssignments == null) {
throw new PolicyViolationException("Attempt add archetypeRef without a matching assignment");
} else {
boolean match = true;
for (PrismReferenceValue archetypeRefDeltaVal : archetypeRefDelta.getValuesToAdd()) {
if (!archetypeOidFromAssignments.equals(archetypeRefDeltaVal.getOid())) {
match = false;
}
}
if (match) {
return;
} else {
throw new PolicyViolationException("Attempt add archetypeRef that does not match assignment");
}
}
}
throw new PolicyViolationException("Attempt to modify archetypeRef directly");
}
}
}

// todo check authorization in this method
private <F extends ObjectType> boolean startTracingIfRequested(LensContext<F> context, Task task,
OperationResultBuilder builder, OperationResult parentResult) throws SchemaException, ObjectNotFoundException,
Expand Down Expand Up @@ -246,6 +291,7 @@ public <F extends ObjectType> LensContext<F> previewChanges(LensContext<F> conte

projector.projectAllWaves(context, "preview", task, result);
clockworkHookHelper.invokePreview(context, task, result);
policyRuleEnforcer.execute(context);
policyRuleSuspendTaskExecutor.execute(context, task, result);

} catch (ConfigurationException | SecurityViolationException | ObjectNotFoundException | SchemaException |
Expand Down Expand Up @@ -365,68 +411,20 @@ public <F extends ObjectType> HookOperationMode click(LensContext<F> context, Ta
context.generateRequestIdentifierIfNeeded();
// We need to do this BEFORE projection. If we would do that after projection
// there will be secondary changes that are not part of the request.
clockworkAuditHelper.audit(context, AuditEventStage.REQUEST, task, result, parentResult); // we need to take the overall ("run" operation result) not the current one
clockworkAuditHelper.audit(context, AuditEventStage.REQUEST, task, result, parentResult); // we need to take the overall ("run" operation result) not the current one
}

boolean recompute = false;
if (!context.isFresh()) {
LOGGER.trace("Context is not fresh -- forcing cleanup and recomputation");
recompute = true;
} else if (context.getExecutionWave() > context.getProjectionWave()) { // should not occur
LOGGER.warn("Execution wave is greater than projection wave -- forcing cleanup and recomputation");
recompute = true;
} else if (state == ModelState.PRIMARY && ModelExecuteOptions.getInitialPartialProcessing(context.getOptions()) != null) {
LOGGER.trace("Initial phase was run with initialPartialProcessing option -- forcing cleanup and recomputation");
recompute = true;
}

if (recompute) {
context.cleanup();
LOGGER.trace("Running projector with cleaned-up context for execution wave {}", context.getExecutionWave());
projector.project(context, "PROJECTOR ("+state+")", task, result);
} else if (context.getExecutionWave() == context.getProjectionWave()) {
LOGGER.trace("Resuming projector for execution wave {}", context.getExecutionWave());
projector.resume(context, "PROJECTOR ("+state+")", task, result);
} else {
LOGGER.trace("Skipping projection because the context is fresh and projection for current wave has already run");
}
projectIfNeeded(context, task, result);

if (!context.isRequestAuthorized()) {
clockworkAuthorizationHelper.authorizeContextRequest(context, task, result);
}

medic.traceContext(LOGGER, "CLOCKWORK (" + state + ")", "before processing", true, context, false);
if (InternalsConfig.consistencyChecks) {
try {
context.checkConsistence();
} catch (IllegalStateException e) {
throw new IllegalStateException(e.getMessage()+" in clockwork, state="+state, e);
}
}
if (InternalsConfig.encryptionChecks && !ModelExecuteOptions.isNoCrypt(context.getOptions())) {
context.checkEncrypted();
}
context.checkConsistenceIfNeeded();
context.checkEncryptedIfNeeded();

switch (state) {
case INITIAL:
processInitialToPrimary(context);
break;
case PRIMARY:
processPrimaryToSecondary(context, task, result);
break;
case SECONDARY:
if (context.getExecutionWave() > context.getMaxWave() + 1) {
processSecondaryToFinal(context, task, result);
} else {
processSecondary(context, task, result, parentResult);
}
break;
case FINAL:
HookOperationMode mode = processFinal(context, task, result, parentResult);
medic.clockworkFinish(context);
return mode;
}
return clockworkHookHelper.invokeHooks(context, task, result);
return moveStateForward(context, task, parentResult, result, state);

} catch (CommunicationException | ConfigurationException | ExpressionEvaluationException | ObjectNotFoundException |
PolicyViolationException | SchemaException | SecurityViolationException | RuntimeException | Error |
Expand All @@ -445,21 +443,76 @@ public <F extends ObjectType> HookOperationMode click(LensContext<F> context, Ta
}
}

private <F extends ObjectType> void projectIfNeeded(LensContext<F> context, Task task, OperationResult result)
throws SchemaException, ConfigurationException, PolicyViolationException, ExpressionEvaluationException,
ObjectNotFoundException, ObjectAlreadyExistsException, CommunicationException, SecurityViolationException,
PreconditionViolationException {
boolean recompute = false;
if (!context.isFresh()) {
LOGGER.trace("Context is not fresh -- forcing cleanup and recomputation");
recompute = true;
} else if (context.getExecutionWave() > context.getProjectionWave()) { // should not occur
LOGGER.warn("Execution wave is greater than projection wave -- forcing cleanup and recomputation");
recompute = true;
} else if (context.isInPrimary() && ModelExecuteOptions.getInitialPartialProcessing(context.getOptions()) != null) {
LOGGER.trace("Initial phase was run with initialPartialProcessing option -- forcing cleanup and recomputation");
recompute = true;
}

if (recompute) {
context.cleanup();
LOGGER.trace("Running projector with cleaned-up context for execution wave {}", context.getExecutionWave());
projector.project(context, "PROJECTOR ("+ context.getState() +")", task, result);
} else if (context.getExecutionWave() == context.getProjectionWave()) {
LOGGER.trace("Resuming projector for execution wave {}", context.getExecutionWave());
projector.resume(context, "PROJECTOR ("+ context.getState() +")", task, result);
} else {
LOGGER.trace("Skipping projection because the context is fresh and projection for current wave has already run");
}
}

private <F extends ObjectType> HookOperationMode moveStateForward(LensContext<F> context, Task task, OperationResult parentResult,
OperationResult result, ModelState state) throws PolicyViolationException, ObjectNotFoundException, SchemaException,
ObjectAlreadyExistsException, CommunicationException, ConfigurationException, SecurityViolationException,
ExpressionEvaluationException, PreconditionViolationException {
switch (state) {
case INITIAL:
processInitialToPrimary(context);
break;
case PRIMARY:
processPrimaryToSecondary(context, task, result);
break;
case SECONDARY:
if (context.getExecutionWave() > context.getMaxWave() + 1) {
processSecondaryToFinal(context, task, result);
} else {
processSecondary(context, task, result, parentResult);
}
break;
case FINAL:
HookOperationMode mode = processFinal(context, task, result, parentResult);
medic.clockworkFinish(context);
return mode;
}
return clockworkHookHelper.invokeHooks(context, task, result);
}

private <F extends ObjectType> void switchState(LensContext<F> context, ModelState newState) {
medic.clockworkStateSwitch(context, newState);
context.setState(newState);
}

private <F extends ObjectType> void processInitialToPrimary(LensContext<F> context) {
// Context loaded, nothing special do. Bump state to PRIMARY.
private <F extends ObjectType> void processInitialToPrimary(LensContext<F> context) throws PolicyViolationException {
// To mimic operation of the original enforcer hook, we execute the following only in the initial state.
policyRuleEnforcer.execute(context);

switchState(context, ModelState.PRIMARY);
}

private <F extends ObjectType> void processPrimaryToSecondary(LensContext<F> context, Task task, OperationResult result) throws PolicyViolationException, ObjectNotFoundException, SchemaException {
// Nothing to do now. The context is already recomputed.
switchState(context, ModelState.SECONDARY);

policyRuleSuspendTaskExecutor.execute(context, task, result);

switchState(context, ModelState.SECONDARY);
}

private <F extends ObjectType> void processSecondary(LensContext<F> context, Task task, OperationResult result, OperationResult overallResult)
Expand All @@ -468,13 +521,13 @@ private <F extends ObjectType> void processSecondary(LensContext<F> context, Tas

Holder<Boolean> restartRequestedHolder = new Holder<>(false);

medic.partialExecute("execution",
medic.partialExecute(Components.EXECUTION,
(result1) -> {
boolean restartRequested = changeExecutor.executeChanges(context, task, result1);
restartRequestedHolder.setValue(restartRequested);
},
context.getPartialProcessingOptions()::getExecution,
Clockwork.class, context, result);
Clockwork.class, context, null, result);

clockworkAuditHelper.audit(context, AuditEventStage.EXECUTION, task, result, overallResult);

Expand Down

0 comments on commit 2c98457

Please sign in to comment.