Skip to content

Commit

Permalink
Add dead nodes handling (MID-5904)
Browse files Browse the repository at this point in the history 
Marking nodes that have not checked-in for 30 seconds as down.
Avoiding REST calls to nodes that are marked as down.
Deleting dead nodes according to specified clean-up policy.
Minor fixes in cleanup task handler and closed task cleanup method.
New XmlTypeConverter.compareMillis method.
  • Loading branch information
@mederly
mederly committed Dec 10, 2019
1 parent cae902c commit 82381c2
Show file tree
Hide file tree
Showing 12 changed files with 367 additions and 200 deletions.
55 changes: 33 additions & 22 deletions infra/prism-api/src/main/java/com/evolveum/midpoint/prism/xml/XmlTypeConverter.java
View file
Expand Up @@ -291,14 +291,25 @@ public static XMLGregorianCalendar addMillis(XMLGregorianCalendar now, long dura
public static int compare(XMLGregorianCalendar o1, XMLGregorianCalendar o2) {
if (o1 == null && o2 == null) {
return 0;
}
if (o1 == null) {
} else if (o1 == null) {
return -1;
} else if (o2 == null) {
return 1;
} else {
return o1.compare(o2);
}
if (o2 == null) {
}

public static int compareMillis(XMLGregorianCalendar o1, XMLGregorianCalendar o2) {
if (o1 == null && o2 == null) {
return 0;
} else if (o1 == null) {
return -1;
} else if (o2 == null) {
return 1;
} else {
return Long.compare(o1.toGregorianCalendar().getTimeInMillis(), o2.toGregorianCalendar().getTimeInMillis());
}
return o1.compare(o2);
}

public static boolean isBeforeNow(XMLGregorianCalendar time) {
Expand Down Expand Up @@ -445,24 +456,24 @@ public static <T> T toJavaValue(String stringContent, Class<T> type, boolean exc
/**
* Parse PolyString from DOM element.
*/
private static PolyString polyStringToJava(Element polyStringElement) throws SchemaException {
Element origElement = DOMUtil.getChildElement(polyStringElement, PrismConstants.POLYSTRING_ELEMENT_ORIG_QNAME);
if (origElement == null) {
// Check for a special syntactic short-cut. If the there are no child elements use the text content of the
// element as the value of orig
if (DOMUtil.hasChildElements(polyStringElement)) {
throw new SchemaException("Missing element "+PrismConstants.POLYSTRING_ELEMENT_ORIG_QNAME+" in polystring element "+
DOMUtil.getQName(polyStringElement));
}
String orig = polyStringElement.getTextContent();
return new PolyString(orig);
private static PolyString polyStringToJava(Element polyStringElement) throws SchemaException {
Element origElement = DOMUtil.getChildElement(polyStringElement, PrismConstants.POLYSTRING_ELEMENT_ORIG_QNAME);
if (origElement == null) {
// Check for a special syntactic short-cut. If the there are no child elements use the text content of the
// element as the value of orig
if (DOMUtil.hasChildElements(polyStringElement)) {
throw new SchemaException("Missing element "+PrismConstants.POLYSTRING_ELEMENT_ORIG_QNAME+" in polystring element "+
DOMUtil.getQName(polyStringElement));
}
String orig = origElement.getTextContent();
String norm = null;
Element normElement = DOMUtil.getChildElement(polyStringElement, PrismConstants.POLYSTRING_ELEMENT_NORM_QNAME);
if (normElement != null) {
norm = normElement.getTextContent();
}
return new PolyString(orig, norm);
String orig = polyStringElement.getTextContent();
return new PolyString(orig);
}
String orig = origElement.getTextContent();
String norm = null;
Element normElement = DOMUtil.getChildElement(polyStringElement, PrismConstants.POLYSTRING_ELEMENT_NORM_QNAME);
if (normElement != null) {
norm = normElement.getTextContent();
}
return new PolyString(orig, norm);
}
}
38 changes: 34 additions & 4 deletions infra/schema/src/main/resources/xml/ns/public/common/common-core-3.xsd
View file
Expand Up @@ -21147,33 +21147,41 @@
</xsd:annotation>
</xsd:element>
<xsd:element name="closedTasks" type="tns:CleanupPolicyType" minOccurs="0">
<xsd:annotation>
<xsd:annotation>
<xsd:appinfo>
<a:displayName>CleanupPoliciesType.closedTasks</a:displayName>
</xsd:appinfo>
</xsd:annotation>
</xsd:element>
<xsd:element name="closedCertificationCampaigns" type="tns:CleanupPolicyType" minOccurs="0">
<xsd:annotation>
<xsd:annotation>
<xsd:appinfo>
<a:displayName>CleanupPoliciesType.closedCertificationCampaigns</a:displayName>
</xsd:appinfo>
</xsd:annotation>
</xsd:element>
<xsd:element name="outputReports" type="tns:CleanupPolicyType" minOccurs="0">
<xsd:annotation>
<xsd:annotation>
<xsd:appinfo>
<a:displayName>CleanupPoliciesType.outputReports</a:displayName>
</xsd:appinfo>
</xsd:annotation>
</xsd:element>
<xsd:element name="objectResults" type="tns:CleanupPolicyType" minOccurs="0">
<xsd:annotation>
<xsd:annotation>
<xsd:appinfo>
<a:displayName>CleanupPoliciesType.objectResults</a:displayName>
</xsd:appinfo>
</xsd:annotation>
</xsd:element>
<xsd:element name="deadNodes" type="tns:DeadNodeCleanupPolicyType" minOccurs="0">
<xsd:annotation>
<xsd:appinfo>
<a:displayName>CleanupPoliciesType.deadNodes</a:displayName>
<a:since>4.1</a:since>
</xsd:appinfo>
</xsd:annotation>
</xsd:element>
</xsd:sequence>
</xsd:complexType>

Expand Down Expand Up @@ -21206,6 +21214,28 @@
</xsd:sequence>
</xsd:complexType>

<xsd:complexType name="DeadNodeCleanupPolicyType">
<xsd:annotation>
<xsd:documentation>
Cleanup policy for dead nodes.
</xsd:documentation>
<xsd:appinfo>
<a:container/>
<a:since>4.1</a:since>
</xsd:appinfo>
</xsd:annotation>
<xsd:sequence>
<xsd:element name="maxAge" type="xsd:duration" minOccurs="0">
<xsd:annotation>
<xsd:appinfo>
<a:displayName>DeadNodeCleanupPolicyType.maxAge</a:displayName>
<a:help>DeadNodeCleanupPolicyType.maxAge.help</a:help>
</xsd:appinfo>
</xsd:annotation>
</xsd:element>
</xsd:sequence>
</xsd:complexType>

<!-- Things relevant to model context serialization - more precisely, those parts that are publicly visible
(i.e. their java counterparts are - or could be - contained in model-api) -->

Expand Down
2 changes: 1 addition & 1 deletion infra/util/src/main/java/com/evolveum/midpoint/util/template/ReferenceResolver.java
View file
Expand Up @@ -26,4 +26,4 @@ public interface ReferenceResolver {
* @return Resolved reference value e.g. "0003"
*/
String resolve(String scope, String reference, @NotNull List<String> parameters);
}
}
141 changes: 73 additions & 68 deletions ...model-impl/src/main/java/com/evolveum/midpoint/model/impl/cleanup/CleanUpTaskHandler.java
View file
Expand Up @@ -22,12 +22,7 @@
import com.evolveum.midpoint.util.exception.SchemaException;
import com.evolveum.midpoint.util.logging.Trace;
import com.evolveum.midpoint.util.logging.TraceManager;
import com.evolveum.midpoint.xml.ns._public.common.common_3.CleanupPoliciesType;
import com.evolveum.midpoint.xml.ns._public.common.common_3.CleanupPolicyType;
import com.evolveum.midpoint.xml.ns._public.common.common_3.SystemConfigurationType;
import com.evolveum.midpoint.xml.ns._public.common.common_3.SystemObjectsType;
import com.evolveum.midpoint.xml.ns._public.common.common_3.TaskPartitionDefinitionType;

import com.evolveum.midpoint.xml.ns._public.common.common_3.*;
import org.jetbrains.annotations.NotNull;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
Expand Down Expand Up @@ -102,85 +97,96 @@ public TaskRunResult run(RunningTask task, TaskPartitionDefinitionType partition
return runResult;
}

CleanupPolicyType auditCleanupPolicy = cleanupPolicies.getAuditRecords();
if (auditCleanupPolicy != null) {
try {
// TODO report progress
auditService.cleanupAudit(auditCleanupPolicy, opResult);
} catch (Exception ex) {
LOGGER.error("Audit cleanup: {}", ex.getMessage(), ex);
opResult.recordFatalError(ex.getMessage(), ex);
runResult.setRunResultStatus(TaskRunResultStatus.PERMANENT_ERROR);
if (task.canRun()) {
CleanupPolicyType auditCleanupPolicy = cleanupPolicies.getAuditRecords();
if (auditCleanupPolicy != null) {
try {
// TODO report progress
auditService.cleanupAudit(auditCleanupPolicy, opResult);
} catch (Exception ex) {
LOGGER.error("Audit cleanup: {}", ex.getMessage(), ex);
opResult.recordFatalError(ex.getMessage(), ex);
runResult.setRunResultStatus(TaskRunResultStatus.PERMANENT_ERROR);
}
} else {
LOGGER.trace("Cleanup: No clean up policy for audit specified. Finishing clean up task.");
}
} else {
LOGGER.trace("Cleanup: No clean up policy for audit specified. Finishing clean up task.");
}

CleanupPolicyType closedTasksPolicy = cleanupPolicies.getClosedTasks();
if (closedTasksPolicy != null) {
try {
taskManager.cleanupTasks(closedTasksPolicy, task, opResult);
} catch (Exception ex) {
LOGGER.error("Tasks cleanup: {}", ex.getMessage(), ex);
opResult.recordFatalError(ex.getMessage(), ex);
runResult.setRunResultStatus(TaskRunResultStatus.PERMANENT_ERROR);
if (task.canRun()) {
CleanupPolicyType closedTasksPolicy = cleanupPolicies.getClosedTasks();
if (closedTasksPolicy != null) {
try {
taskManager.cleanupTasks(closedTasksPolicy, task, opResult);
} catch (Exception ex) {
LOGGER.error("Tasks cleanup: {}", ex.getMessage(), ex);
opResult.recordFatalError(ex.getMessage(), ex);
runResult.setRunResultStatus(TaskRunResultStatus.PERMANENT_ERROR);
}
} else {
LOGGER.trace("Cleanup: No clean up policy for closed tasks specified. Finishing clean up task.");
}
} else {
LOGGER.trace("Cleanup: No clean up policy for closed tasks specified. Finishing clean up task.");
}

CleanupPolicyType reportCleanupPolicy = cleanupPolicies.getOutputReports();
if (reportCleanupPolicy != null) {
try {
if (reportManager == null) {
//TODO improve dependencies for report-impl (probably for tests) and set autowire to required
LOGGER.error("Report manager was not autowired, reports cleanup will be skipped.");
} else {
// TODO report progress
reportManager.cleanupReports(reportCleanupPolicy, opResult);
if (task.canRun()) {
DeadNodeCleanupPolicyType deadNodesPolicy = cleanupPolicies.getDeadNodes();
if (deadNodesPolicy != null) {
try {
taskManager.cleanupNodes(deadNodesPolicy, task, opResult);
} catch (Exception ex) {
LOGGER.error("Nodes cleanup: {}", ex.getMessage(), ex);
opResult.recordFatalError(ex.getMessage(), ex);
runResult.setRunResultStatus(TaskRunResultStatus.PERMANENT_ERROR);
}
} catch (Exception ex) {
LOGGER.error("Reports cleanup: {}", ex.getMessage(), ex);
opResult.recordFatalError(ex.getMessage(), ex);
runResult.setRunResultStatus(TaskRunResultStatus.PERMANENT_ERROR);
} else {
LOGGER.trace("Cleanup: No clean up policy for closed tasks specified. Finishing clean up task.");
}
} else {
LOGGER.trace("Cleanup: No clean up policy for report specified. Finishing clean up task.");
}

CleanupPolicyType closedCampaignsPolicy = cleanupPolicies.getClosedCertificationCampaigns();
if (closedCampaignsPolicy != null) {
try {
certificationService.cleanupCampaigns(closedCampaignsPolicy, task, opResult);
} catch (Throwable ex) {
LOGGER.error("Campaigns cleanup: {}", ex.getMessage(), ex);
opResult.recordFatalError(ex.getMessage(), ex);
runResult.setRunResultStatus(TaskRunResultStatus.PERMANENT_ERROR);
if (task.canRun()) {
CleanupPolicyType reportCleanupPolicy = cleanupPolicies.getOutputReports();
if (reportCleanupPolicy != null) {
try {
if (reportManager == null) {
//TODO improve dependencies for report-impl (probably for tests) and set autowire to required
LOGGER.error("Report manager was not autowired, reports cleanup will be skipped.");
} else {
// TODO report progress
reportManager.cleanupReports(reportCleanupPolicy, opResult);
}
} catch (Exception ex) {
LOGGER.error("Reports cleanup: {}", ex.getMessage(), ex);
opResult.recordFatalError(ex.getMessage(), ex);
runResult.setRunResultStatus(TaskRunResultStatus.PERMANENT_ERROR);
}
} else {
LOGGER.trace("Cleanup: No clean up policy for report specified. Finishing clean up task.");
}
} else {
LOGGER.trace("Cleanup: No clean up policy for closed tasks specified. Finishing clean up task.");
}

if (task.canRun()) {
CleanupPolicyType closedCampaignsPolicy = cleanupPolicies.getClosedCertificationCampaigns();
if (closedCampaignsPolicy != null) {
try {
certificationService.cleanupCampaigns(closedCampaignsPolicy, task, opResult);
} catch (Throwable ex) {
LOGGER.error("Campaigns cleanup: {}", ex.getMessage(), ex);
opResult.recordFatalError(ex.getMessage(), ex);
runResult.setRunResultStatus(TaskRunResultStatus.PERMANENT_ERROR);
}
} else {
LOGGER.trace("Cleanup: No clean up policy for closed tasks specified. Finishing clean up task.");
}
}

opResult.computeStatus();
// This "run" is finished. But the task goes on ...
runResult.setRunResultStatus(TaskRunResultStatus.FINISHED);
opResult.computeStatusIfUnknown();
if (runResult.getRunResultStatus() == null) {
runResult.setRunResultStatus(TaskRunResultStatus.FINISHED);
}
LOGGER.trace("CleanUpTaskHandler.run stopping");
return runResult;
}

@Override
public Long heartbeat(Task task) {
// TODO Auto-generated method stub
return null;
}

@Override
public void refreshStatus(Task task) {
// TODO Auto-generated method stub

}

@Override
public String getCategoryName(Task task) {
if (task != null && task.getExtensionContainerRealValueOrClone(SchemaConstants.MODEL_EXTENSION_CLEANUP_POLICIES) != null) {
Expand All @@ -194,5 +200,4 @@ public String getCategoryName(Task task) {
public List<String> getCategoryNames() {
return Arrays.asList(TaskCategory.UTIL, TaskCategory.SYSTEM);
}

}
4 changes: 3 additions & 1 deletion model/report-impl/src/main/java/com/evolveum/midpoint/report/impl/ReportManagerImpl.java
View file
Expand Up @@ -361,7 +361,9 @@ public void deleteReportOutput(ReportOutputType reportOutput, OperationResult pa
File file = new File(filePath);

if (file.exists()) {
file.delete();
if (!file.delete()) {
LOGGER.error("Couldn't delete report file {}", file);
}
} else {
String fileName = checkNodeAndFileName(file, reportOutput, result);
if (fileName == null) {
Expand Down
5 changes: 5 additions & 0 deletions repo/task-api/src/main/java/com/evolveum/midpoint/task/api/TaskManager.java
View file
Expand Up @@ -297,6 +297,11 @@ void modifyTask(String oid, Collection<? extends ItemDelta> modifications, Opera
*/
void cleanupTasks(CleanupPolicyType closedTasksPolicy, RunningTask task, OperationResult opResult) throws SchemaException;

/**
* Deletes dead nodes, i.e. ones that were not checked-in for a given time period.
*/
void cleanupNodes(DeadNodeCleanupPolicyType deadNodesPolicy, RunningTask task, OperationResult opResult) throws SchemaException;

/**
* This is a signal to task manager that a new task was created in the repository.
* Task manager can react to it e.g. by creating shadow quartz job and trigger.
Expand Down
7 changes: 6 additions & 1 deletion .../task-quartz-impl/src/main/java/com/evolveum/midpoint/task/quartzimpl/NodeIdComputer.java
View file
Expand Up @@ -10,6 +10,7 @@
import com.evolveum.midpoint.common.configuration.api.MidpointConfiguration;
import com.evolveum.midpoint.prism.PrismContext;
import com.evolveum.midpoint.prism.PrismObject;
import com.evolveum.midpoint.prism.xml.XmlTypeConverter;
import com.evolveum.midpoint.repo.api.RepositoryService;
import com.evolveum.midpoint.schema.SearchResultList;
import com.evolveum.midpoint.schema.result.OperationResult;
Expand Down Expand Up @@ -161,8 +162,12 @@ private String getNodeIdFromExpression(String expression, OperationResult result
try {
// Let us try to create node with given name. If we fail we know we need to iterate.
// If we succeed, we will (later) replace the node with the correct content.
// Note that we set (fake) last check-in time here so this node will not be accidentally cleaned-up.
// TODO consider moving this addObject call to NodeRegistrar (requires cleanup of the mix of
// Spring injected and manually created objects)
NodeType node = new NodeType(prismContext)
.name(candidateNodeId);
.name(candidateNodeId)
.lastCheckInTime(XmlTypeConverter.createXMLGregorianCalendar());
repositoryService.addObject(node.asPrismObject(), null, result);
} catch (ObjectAlreadyExistsException e) {
// We have a conflict. But the node might be - in fact - dead. So let's try to reclaim it if possible.
Expand Down

0 comments on commit 82381c2

Please sign in to comment.