Skip to content

Commit

Permalink
Fixed MID-3585 (livesync vs stalled) + greatly simplified updating pr…
Browse files Browse the repository at this point in the history 
…ogress + operation stats in tasks.
  • Loading branch information
@mederly
mederly committed Nov 3, 2017
1 parent 36d5c00 commit 38b82ec
Show file tree
Hide file tree
Showing 31 changed files with 1,945 additions and 1,913 deletions.
39 changes: 15 additions & 24 deletions .../evolveum/midpoint/certification/impl/AccessCertificationCampaignCreationTaskHandler.java
View file
Expand Up @@ -16,19 +16,15 @@

package com.evolveum.midpoint.certification.impl;

import com.evolveum.midpoint.prism.PrismContext;
import com.evolveum.midpoint.schema.result.OperationResult;
import com.evolveum.midpoint.schema.util.ObjectTypeUtil;
import com.evolveum.midpoint.task.api.Task;
import com.evolveum.midpoint.task.api.TaskCategory;
import com.evolveum.midpoint.task.api.TaskHandler;
import com.evolveum.midpoint.task.api.TaskManager;
import com.evolveum.midpoint.task.api.TaskRunResult;
import com.evolveum.midpoint.task.api.*;
import com.evolveum.midpoint.task.api.TaskRunResult.TaskRunResultStatus;
import com.evolveum.midpoint.util.logging.LoggingUtils;
import com.evolveum.midpoint.util.logging.Trace;
import com.evolveum.midpoint.util.logging.TraceManager;
import com.evolveum.midpoint.xml.ns._public.common.common_3.AccessCertificationCampaignType;
import org.jetbrains.annotations.NotNull;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

Expand All @@ -43,18 +39,11 @@
@Component
public class AccessCertificationCampaignCreationTaskHandler implements TaskHandler {

public static final String HANDLER_URI = AccessCertificationConstants.NS_CERTIFICATION_TASK_PREFIX + "/campaign-creation/handler-3";
private static final String HANDLER_URI = AccessCertificationConstants.NS_CERTIFICATION_TASK_PREFIX + "/campaign-creation/handler-3";
private static final String CLASS_DOT = AccessCertificationCampaignCreationTaskHandler.class.getName() + ".";

public static final String CLASS_DOT = AccessCertificationCampaignCreationTaskHandler.class.getName() + ".";

@Autowired
private TaskManager taskManager;

@Autowired
private PrismContext prismContext;

@Autowired
private CertificationManagerImpl certificationManager;
@Autowired private TaskManager taskManager;
@Autowired private CertificationManagerImpl certificationManager;

private static final transient Trace LOGGER = TraceManager.getTrace(AccessCertificationCampaignCreationTaskHandler.class);

Expand All @@ -63,11 +52,18 @@ private void initialize() {
taskManager.registerHandler(HANDLER_URI, this);
}

@NotNull
@Override
public StatisticsCollectionStrategy getStatisticsCollectionStrategy() {
return new StatisticsCollectionStrategy()
.fromStoredValues()
.maintainIterationStatistics();
}

@Override
public TaskRunResult run(Task task) {
LOGGER.trace("Task run starting");

long progress = task.getProgress();
OperationResult opResult = new OperationResult(CLASS_DOT+"run");
opResult.setSummarizeSuccesses(true);
TaskRunResult runResult = new TaskRunResult();
Expand All @@ -87,8 +83,6 @@ public TaskRunResult run(Task task) {
String campaignName = null;
String campaignOid = null;
try {
task.startCollectingOperationStatsFromStoredValues(true, false, false);

LOGGER.info("Creating campaign with definition of {}", definitionOid);
AccessCertificationCampaignType campaign = certificationManager.createCampaign(definitionOid, task, opResult);
LOGGER.info("Campaign {} was created.", ObjectTypeUtil.toShortString(campaign));
Expand All @@ -106,7 +100,7 @@ public TaskRunResult run(Task task) {

opResult.computeStatus();
runResult.setRunResultStatus(TaskRunResultStatus.FINISHED);
runResult.setProgress(progress+1);
runResult.setProgress(task.getProgress()+1);
return runResult;

} catch (Exception e) { // TODO better error handling
Expand All @@ -116,10 +110,7 @@ public TaskRunResult run(Task task) {
LoggingUtils.logException(LOGGER, "Error while executing 'create campaign' task handler", e);
opResult.recordFatalError("Error while executing 'create campaign' task handler: "+e.getMessage(), e);
runResult.setRunResultStatus(TaskRunResultStatus.PERMANENT_ERROR);
runResult.setProgress(progress);
return runResult;
} finally {
task.storeOperationStats();
}
}

Expand Down
35 changes: 27 additions & 8 deletions .../java/com/evolveum/midpoint/certification/impl/AccessCertificationClosingTaskHandler.java
View file
Expand Up @@ -76,11 +76,18 @@ private void initialize() {
taskManager.registerHandler(HANDLER_URI, this);
}

@NotNull
@Override
public StatisticsCollectionStrategy getStatisticsCollectionStrategy() {
return new StatisticsCollectionStrategy()
.fromZero();
// implement iteration statistics when needed (along with expected total)
}

@Override
public TaskRunResult run(Task task) {
LOGGER.info("Task run starting");

long progress = task.getProgress();
OperationResult opResult = new OperationResult(CLASS_DOT+"run");
opResult.setSummarizeSuccesses(true);
TaskRunResult runResult = new TaskRunResult();
Expand All @@ -104,29 +111,32 @@ public TaskRunResult run(Task task) {
} catch (ObjectNotFoundException|SchemaException e) {
opResult.computeStatus();
runResult.setRunResultStatus(TaskRunResultStatus.PERMANENT_ERROR);
runResult.setProgress(progress);
LoggingUtils.logUnexpectedException(LOGGER, "Closing task couldn't start for campaign {} because of unexpected exception", e, campaignOid);
return runResult;
}

RunContext runContext = new RunContext();
RunContext runContext = new RunContext(task);
caseList.forEach(aCase -> prepareMetadataDeltas(aCase, campaign, runContext, opResult));
runContext.objectContextMap.forEach((oid, ctx) -> applyMetadataDeltas(ctx, opResult));
runContext.objectContextMap.forEach((oid, ctx) -> applyMetadataDeltas(ctx, runContext, opResult));

opResult.computeStatus();
runResult.setRunResultStatus(TaskRunResultStatus.FINISHED);
runResult.setProgress(progress);
LOGGER.info("Task run stopping (campaign {})", toShortString(campaign));
return runResult;
}

private void applyMetadataDeltas(ObjectContext objectCtx, OperationResult opResult) {
private void applyMetadataDeltas(ObjectContext objectCtx,
RunContext runContext,
OperationResult opResult) {
ObjectType object = objectCtx.object;
List<ItemDelta<?, ?>> deltas = objectCtx.modifications;
try {
LOGGER.info("### Updating {} with:\n{}", toShortString(object), DebugUtil.debugDump(deltas));
if (LOGGER.isTraceEnabled()) {
LOGGER.trace("### Updating {} with:\n{}", toShortString(object), DebugUtil.debugDump(deltas));
}
if (!deltas.isEmpty()) {
repositoryService.modifyObject(object.getClass(), object.getOid(), deltas, opResult);
runContext.task.incrementProgressAndStoreStatsIfNeeded();
}
} catch (ObjectNotFoundException|SchemaException|ObjectAlreadyExistsException e) {
LoggingUtils.logUnexpectedException(LOGGER, "Couldn't update certification metadata for {}", e, toShortString(object));
Expand All @@ -135,6 +145,10 @@ private void applyMetadataDeltas(ObjectContext objectCtx, OperationResult opResu

private void prepareMetadataDeltas(AccessCertificationCaseType aCase, AccessCertificationCampaignType campaign,
RunContext runContext, OperationResult result) {

// we count progress for each certification case and then for each object updated
runContext.task.incrementProgressAndStoreStatsIfNeeded();

String objectOid = aCase.getObjectRef() != null ? aCase.getObjectRef().getOid() : null;
if (objectOid == null) {
LOGGER.error("No object OID in certification case {}: skipping metadata recording", aCase);
Expand Down Expand Up @@ -259,12 +273,17 @@ private class ObjectContext {
@NotNull final ObjectType object;
@NotNull final List<ItemDelta<?, ?>> modifications = new ArrayList<>();

public ObjectContext(@NotNull ObjectType object) {
ObjectContext(@NotNull ObjectType object) {
this.object = object;
}
}

private class RunContext {
final Task task;
final Map<String, ObjectContext> objectContextMap = new HashMap<>();

RunContext(Task task) {
this.task = task;
}
}
}
20 changes: 11 additions & 9 deletions ...a/com/evolveum/midpoint/certification/impl/AccessCertificationRemediationTaskHandler.java
View file
Expand Up @@ -22,11 +22,7 @@
import com.evolveum.midpoint.schema.result.OperationResultStatus;
import com.evolveum.midpoint.schema.util.CertCampaignTypeUtil;
import com.evolveum.midpoint.schema.util.ObjectTypeUtil;
import com.evolveum.midpoint.task.api.Task;
import com.evolveum.midpoint.task.api.TaskCategory;
import com.evolveum.midpoint.task.api.TaskHandler;
import com.evolveum.midpoint.task.api.TaskManager;
import com.evolveum.midpoint.task.api.TaskRunResult;
import com.evolveum.midpoint.task.api.*;
import com.evolveum.midpoint.task.api.TaskRunResult.TaskRunResultStatus;
import com.evolveum.midpoint.util.exception.ObjectNotFoundException;
import com.evolveum.midpoint.util.exception.SchemaException;
Expand All @@ -38,6 +34,7 @@
import com.evolveum.midpoint.xml.ns._public.common.common_3.SystemObjectsType;
import com.evolveum.midpoint.xml.ns._public.common.common_3.UserType;
import com.evolveum.prism.xml.ns._public.types_3.PolyStringType;
import org.jetbrains.annotations.NotNull;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Component;
Expand Down Expand Up @@ -71,11 +68,18 @@ private void initialize() {
taskManager.registerHandler(HANDLER_URI, this);
}

@NotNull
@Override
public StatisticsCollectionStrategy getStatisticsCollectionStrategy() {
return new StatisticsCollectionStrategy()
.fromZero();
// implement iteration stats when needed
}

@Override
public TaskRunResult run(Task task) {
LOGGER.trace("Task run starting");

long progress = task.getProgress();
OperationResult opResult = new OperationResult(CLASS_DOT+"run");
opResult.setSummarizeSuccesses(true);
TaskRunResult runResult = new TaskRunResult();
Expand Down Expand Up @@ -120,7 +124,7 @@ public TaskRunResult run(Task task) {
caseHelper.markCaseAsRemedied(campaignOid, caseId, task, caseResult);
caseResult.computeStatus();
revokedOk++;
progress++;
task.incrementProgressAndStoreStatsIfNeeded();
} catch (Exception e) { // TODO
String message = "Couldn't revoke case " + caseId + ": " + e.getMessage();
LoggingUtils.logUnexpectedException(LOGGER, message, e);
Expand All @@ -137,15 +141,13 @@ public TaskRunResult run(Task task) {
certificationManager.closeCampaign(campaignOid, task, opResult);

runResult.setRunResultStatus(TaskRunResultStatus.FINISHED);
runResult.setProgress(progress);
LOGGER.trace("Task run stopping (campaign {})", ObjectTypeUtil.toShortString(campaign));
return runResult;

} catch (Exception e) { // TODO better error handling
LoggingUtils.logException(LOGGER, "Error while executing remediation task handler", e);
opResult.recordFatalError("Error while executing remediation task handler: "+e.getMessage(), e);
runResult.setRunResultStatus(TaskRunResultStatus.PERMANENT_ERROR);
runResult.setProgress(progress);
return runResult;
}
}
Expand Down
40 changes: 14 additions & 26 deletions ...model-impl/src/main/java/com/evolveum/midpoint/model/impl/cleanup/CleanUpTaskHandler.java
View file
Expand Up @@ -18,18 +18,13 @@

import com.evolveum.midpoint.audit.api.AuditService;
import com.evolveum.midpoint.model.api.ModelPublicConstants;
import com.evolveum.midpoint.prism.PrismContext;
import com.evolveum.midpoint.prism.PrismObject;
import com.evolveum.midpoint.repo.api.RepositoryService;
import com.evolveum.midpoint.report.api.ReportManager;
import com.evolveum.midpoint.schema.constants.SchemaConstants;
import com.evolveum.midpoint.schema.result.OperationConstants;
import com.evolveum.midpoint.schema.result.OperationResult;
import com.evolveum.midpoint.task.api.Task;
import com.evolveum.midpoint.task.api.TaskCategory;
import com.evolveum.midpoint.task.api.TaskHandler;
import com.evolveum.midpoint.task.api.TaskManager;
import com.evolveum.midpoint.task.api.TaskRunResult;
import com.evolveum.midpoint.task.api.*;
import com.evolveum.midpoint.task.api.TaskRunResult.TaskRunResultStatus;
import com.evolveum.midpoint.util.exception.ObjectNotFoundException;
import com.evolveum.midpoint.util.exception.SchemaException;
Expand All @@ -39,6 +34,7 @@
import com.evolveum.midpoint.xml.ns._public.common.common_3.CleanupPolicyType;
import com.evolveum.midpoint.xml.ns._public.common.common_3.SystemConfigurationType;
import com.evolveum.midpoint.xml.ns._public.common.common_3.SystemObjectsType;
import org.jetbrains.annotations.NotNull;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

Expand All @@ -54,7 +50,6 @@ public class CleanUpTaskHandler implements TaskHandler {
@Autowired private TaskManager taskManager;
@Autowired private RepositoryService repositoryService;
@Autowired private AuditService auditService;
@Autowired private PrismContext prismContext;

@Autowired(required = false)
private ReportManager reportManager;
Expand All @@ -66,20 +61,18 @@ private void initialize() {
taskManager.registerHandler(HANDLER_URI, this);
}

@NotNull
@Override
public TaskRunResult run(Task task) {
task.startCollectingOperationStatsFromZero(true, false, false);
try {
return runInternal(task);
} finally {
task.storeOperationStats();
}
public StatisticsCollectionStrategy getStatisticsCollectionStrategy() {
return new StatisticsCollectionStrategy()
.fromZero()
.maintainIterationStatistics();
}

private TaskRunResult runInternal(Task task) {
@Override
public TaskRunResult run(Task task) {
LOGGER.trace("CleanUpTaskHandler.run starting");

long progress = task.getProgress();
OperationResult opResult = new OperationResult(OperationConstants.CLEANUP);
TaskRunResult runResult = new TaskRunResult();
runResult.setOperationResult(opResult);
Expand All @@ -96,13 +89,11 @@ private TaskRunResult runInternal(Task task) {
LOGGER.error("Cleanup: Object does not exist: {}", ex.getMessage(), ex);
opResult.recordFatalError("Object does not exist: " + ex.getMessage(), ex);
runResult.setRunResultStatus(TaskRunResultStatus.PERMANENT_ERROR);
runResult.setProgress(progress);
return runResult;
} catch (SchemaException ex) {
LOGGER.error("Cleanup: Error dealing with schema: {}", ex.getMessage(), ex);
opResult.recordFatalError("Error dealing with schema: " + ex.getMessage(), ex);
runResult.setRunResultStatus(TaskRunResultStatus.PERMANENT_ERROR);
runResult.setProgress(progress);
return runResult;
}
SystemConfigurationType systemConfigType = systemConfig.asObjectable();
Expand All @@ -113,21 +104,20 @@ private TaskRunResult runInternal(Task task) {
LOGGER.trace("Cleanup: No clean up polices specified. Finishing clean up task.");
opResult.computeStatus();
runResult.setRunResultStatus(TaskRunResultStatus.FINISHED);
runResult.setProgress(progress);
return runResult;
}

CleanupPolicyType auditCleanupPolicy = cleanupPolicies.getAuditRecords();
if (auditCleanupPolicy != null) {
try {
// TODO report progress
auditService.cleanupAudit(auditCleanupPolicy, opResult);
} catch (Exception ex) {
LOGGER.error("Cleanup: {}", ex.getMessage(), ex);
opResult.recordFatalError(ex.getMessage(), ex);
runResult.setRunResultStatus(TaskRunResultStatus.PERMANENT_ERROR);
runResult.setProgress(progress);
}
} else{
} else {
LOGGER.trace("Cleanup: No clean up policy for audit specified. Finishing clean up task.");
}

Expand All @@ -139,9 +129,8 @@ private TaskRunResult runInternal(Task task) {
LOGGER.error("Cleanup: {}", ex.getMessage(), ex);
opResult.recordFatalError(ex.getMessage(), ex);
runResult.setRunResultStatus(TaskRunResultStatus.PERMANENT_ERROR);
runResult.setProgress(progress);
}
} else{
} else {
LOGGER.trace("Cleanup: No clean up policy for closed tasks specified. Finishing clean up task.");
}

Expand All @@ -152,21 +141,20 @@ private TaskRunResult runInternal(Task task) {
//TODO improve dependencies for report-impl (probably for tests) and set autowire to required
LOGGER.error("Report manager was not autowired, reports cleanup will be skipped.");
} else {
// TODO report progress
reportManager.cleanupReports(reportCleanupPolicy, opResult);
}
} catch (Exception ex) {
LOGGER.error("Cleanup: {}", ex.getMessage(), ex);
opResult.recordFatalError(ex.getMessage(), ex);
runResult.setRunResultStatus(TaskRunResultStatus.PERMANENT_ERROR);
runResult.setProgress(progress);
}
} else{
} else {
LOGGER.trace("Cleanup: No clean up policy for report specified. Finishing clean up task.");
}
opResult.computeStatus();
// This "run" is finished. But the task goes on ...
runResult.setRunResultStatus(TaskRunResultStatus.FINISHED);
runResult.setProgress(progress);
LOGGER.trace("CleanUpTaskHandler.run stopping");
return runResult;
}
Expand Down
1 change: 0 additions & 1 deletion ...main/java/com/evolveum/midpoint/model/impl/importer/ImportObjectsFromFileTaskHandler.java
View file
Expand Up @@ -160,7 +160,6 @@ public TaskRunResult run(Task task) {
OperationResult opResult = task.getResult().createSubresult(OperationConstants.IMPORT_OBJECTS_FROM_FILE);
TaskRunResult runResult = new TaskRunResult();
runResult.setOperationResult(opResult);
runResult.setProgress(0);

// Determine the input file from task extension

Expand Down

0 comments on commit 38b82ec

Please sign in to comment.