Skip to content

Commit

Permalink
Improve trace processing
Browse files Browse the repository at this point in the history
1) Add ninja option to analyze and transform traces
2) Improve visualization of assignment traces
  • Loading branch information
mederly committed Sep 3, 2020
1 parent 0b4307b commit 88bd8b2
Show file tree
Hide file tree
Showing 23 changed files with 1,086 additions and 147 deletions.
Expand Up @@ -607,7 +607,7 @@ public int getMappingsCount() {
}

public Integer getAssignmentEvaluationsCount() {
return (int) getChildrenStream(2)
return (int) getChildrenStream(3)
.filter(child -> child.getKind() == OperationKindType.ASSIGNMENT_EVALUATION)
.count();
}
Expand Down Expand Up @@ -639,4 +639,8 @@ public OpNodePresentation getPresentation() {
public void setPresentation(OpNodePresentation presentation) {
this.presentation = presentation;
}

public void resolveReferenceTargetNames(OpNodeTreeBuilder.NameResolver nameResolver) {
}

}
Expand Up @@ -45,6 +45,10 @@ public static OpNode createOpNode(PrismContext prismContext, OperationResultType
case PROJECTOR_TEMPLATE_AFTER_ASSIGNMENTS:
case PROJECTOR_COMPONENT_OTHER:
return new ProjectorComponentOpNode(prismContext, result, info, parent, traceInfo);
case ASSIGNMENT_EVALUATION:
return new AssignmentEvaluationOpNode(prismContext, result, info, parent, traceInfo);
case ASSIGNMENT_SEGMENT_EVALUATION:
return new AssignmentSegmentEvaluationOpNode(prismContext, result, info, parent, traceInfo);
}
}
return new OpNode(prismContext, result, info, parent, traceInfo);
Expand Down
Expand Up @@ -12,6 +12,8 @@
import com.evolveum.midpoint.xml.ns._public.common.common_3.OperationResultType;
import com.evolveum.midpoint.xml.ns._public.common.common_3.TracingOutputType;

import com.evolveum.prism.xml.ns._public.types_3.PolyStringType;

import org.jetbrains.annotations.NotNull;

import java.util.ArrayList;
Expand All @@ -23,25 +25,35 @@ public class OpNodeTreeBuilder {

@NotNull private final PrismContext prismContext;

private IdentityHashMap<OperationResultType, OpResultInfo> infoMap = new IdentityHashMap<>();
private final IdentityHashMap<OperationResultType, OpResultInfo> infoMap = new IdentityHashMap<>();

public OpNodeTreeBuilder(PrismContext prismContext) {
public OpNodeTreeBuilder(@NotNull PrismContext prismContext) {
this.prismContext = prismContext;
}

public List<OpNode> build(TracingOutputType tracingOutput) {
return build(tracingOutput, null);
}

public List<OpNode> build(TracingOutputType tracingOutput, NameResolver nameResolver) {
List<OpNode> rv = new ArrayList<>();
addNode(null, rv, tracingOutput.getResult(), new TraceInfo(tracingOutput));
addNode(null, rv, tracingOutput.getResult(), new TraceInfo(tracingOutput), nameResolver);
return rv;

}

private void addNode(OpNode parent, List<OpNode> rv, OperationResultType result, TraceInfo traceInfo) {
private void addNode(OpNode parent, List<OpNode> rv, OperationResultType result, TraceInfo traceInfo, NameResolver nameResolver) {
OpResultInfo info = OpResultInfo.create(result, infoMap);
OpNode newNode = OpNodeFactory.createOpNode(prismContext, result, info, parent, traceInfo);
if (nameResolver != null) {
newNode.resolveReferenceTargetNames(nameResolver);
}
rv.add(newNode);
for (OperationResultType child : result.getPartialResults()) {
addNode(newNode, newNode.getChildren(), child, traceInfo);
addNode(newNode, newNode.getChildren(), child, traceInfo, nameResolver);
}
}

public interface NameResolver {
PolyStringType getName(String oid);
}
}
Expand Up @@ -59,9 +59,17 @@ public enum OpType {
"com.evolveum.midpoint.model.impl.lens.projector.Projector.assignments",
"Assignments (${m:getAssignmentEvaluationsCount})"),

ASSIGNMENT_EVALUATION(OperationKindType.ASSIGNMENT_EVALUATION, "Assignment evaluation",
ASSIGNMENT_EVALUATION_OUTER(OperationKindType.OTHER, "Assignment evaluation (outer)",
"com.evolveum.midpoint.model.impl.lens.projector.focus.AssignmentTripleEvaluator.evaluateAssignment",
"Assignment evaluation (→ ${c:assignmentTargetName})"),
"Assignment evaluation: → ${c:assignmentTargetName}"),

ASSIGNMENT_EVALUATION(OperationKindType.ASSIGNMENT_EVALUATION, "Assignment evaluation",
"com.evolveum.midpoint.model.impl.lens.AssignmentEvaluator.evaluate",
"Assignment evaluation: ${m:getAssignmentInfo}"),

ASSIGNMENT_SEGMENT_EVALUATION(OperationKindType.OTHER, "Assignment segment evaluation",
"com.evolveum.midpoint.model.impl.lens.AssignmentEvaluator.evaluateFromSegment",
"${m:getSegmentLabel}"),

PROJECTION_ACTIVATION(OperationKindType.OTHER,"Projection activation",
"com.evolveum.midpoint.model.impl.lens.projector.ActivationProcessor.projectionActivation",
Expand Down Expand Up @@ -192,7 +200,8 @@ private static boolean isLoadedFromRepository(OperationResultType result) {
public String getFormattedName(OpNode node) {

if (nameTemplate != null) {
return expandTemplate(node);
return new TemplateExpander()
.expandTemplate(node, nameTemplate);
}

OperationResultType opResult = node.getResult();
Expand All @@ -214,12 +223,6 @@ public String getFormattedName(OpNode node) {
String srcName = getContext(opResult, "segmentSourceName");
String tgtName = getContext(opResult, "segmentTargetName");
return "Segment: " + (srcName != null ? srcName + " " : "") + " → " + (tgtName != null ? " " + tgtName : "");
} else if ("com.evolveum.midpoint.model.impl.lens.AssignmentEvaluator.evaluate".equals(operation)) {
String tgtName = getContext(opResult, "assignmentTargetName");
return "AssignmentEvaluator.evaluate" + (tgtName != null ? " (→ " + tgtName + ")" : "");
} else if ("com.evolveum.midpoint.model.impl.lens.projector.focus.AssignmentTripleEvaluator.evaluateAssignment".equals(operation)) {
String tgtName = getContext(opResult, "assignmentTargetName");
return "AssignmentTripleEvaluator.evaluateAssignment" + (tgtName != null ? " (→ " + tgtName + ")" : "");
} else if ("com.evolveum.midpoint.model.impl.lens.projector.policy.PolicyRuleProcessor.evaluateRule".equals(operation)) {
String triggeredString = getReturn(opResult, "triggered");
int enabledActions = TraceUtil.getReturnsAsStringList(opResult, "enabledActions").size();
Expand Down Expand Up @@ -261,109 +264,6 @@ public String getFormattedName(OpNode node) {
return opResult.getOperation() + (qualifiers.isEmpty() ? "" : " (" + qualifiers + ")");
}

private String expandTemplate(OpNode node) {
return new StringSubstitutor(createResolver(node), "${", "}", '\\')
.replace(nameTemplate);
}

private StringLookup createResolver(OpNode node) {
return spec -> {
String prefix;
String suffix;
String key;

String[] parts = spec.split(":");
if (parts.length == 1) {
prefix = "";
key = spec;
suffix = "";
} else if (parts.length == 2) {
prefix = parts[0];
key = parts[1];
suffix = "";
} else if (parts.length == 3) {
prefix = parts[0];
key = parts[1];
suffix = parts[2];
} else {
return "???";
}

List<String> values = new ArrayList<>();
if (prefix.isEmpty() || prefix.equals("p")) {
collectMatchingParams(values, key, node.getResult().getParams());
}
if (prefix.isEmpty() || prefix.equals("c")) {
collectMatchingParams(values, key, node.getResult().getContext());
}
if (prefix.isEmpty() || prefix.equals("r")) {
collectMatchingParams(values, key, node.getResult().getReturns());
}
if (prefix.isEmpty() || prefix.equals("t")) {
collectMatchingValues(values, key, node.getResult().getTrace());
}
if (prefix.equals("m")) {
collectFromMethod(values, key, node);
}
return String.join(", ", postprocess(values, suffix));
};
}

private void collectFromMethod(List<String> values, String key, OpNode node) {
try {
Object rv = MethodUtils.invokeExactMethod(node, key);
if (rv instanceof Collection) {
for (Object o : (Collection) rv) {
values.add(String.valueOf(o));
}
} else if (rv != null) {
values.add(String.valueOf(rv));
}
} catch (Throwable t) {
values.add("??? " + t.getMessage());
}
}

private void collectMatchingValues(List<String> values, String path, List<TraceType> traces) {
UniformItemPath itemPath = ItemPathParserTemp.parseFromString(path); // FIXME (hack)
for (TraceType trace : traces) {
PrismProperty property = trace.asPrismContainerValue().findProperty(itemPath);
if (property != null) {
for (Object realValue : property.getRealValues()) {
values.add(String.valueOf(realValue));
}
}
}
}

private void collectMatchingParams(List<String> values, String key, ParamsType params) {
int colon = key.indexOf(':');
String processing;
String realKey;
if (colon >= 0) {
realKey = key.substring(0, colon);
processing = key.substring(colon + 1);
} else {
realKey = key;
processing = "";
}
List<String> rawStringValues = asStringList(selectByKey(params, realKey));
values.addAll(postprocess(rawStringValues, processing));
}

private List<String> postprocess(List<String> raw, String processing) {
return raw.stream()
.map(s -> postprocess(s, processing))
.collect(Collectors.toList());
}

private String postprocess(String string, String processing) {
if (processing.contains("L")) {
string = string.toLowerCase();
}
return string;
}

private String getRepoCacheOpDescription(OpNode node, OperationResultType opResult, String last, String commaQualifiers) {
String postfix = "";
if ("getObject".equals(last)) {
Expand Down
@@ -0,0 +1,133 @@
/*
* Copyright (c) 2020 Evolveum and contributors
*
* This work is dual-licensed under the Apache License 2.0
* and European Union Public License. See LICENSE file for details.
*/

package com.evolveum.midpoint.schema.traces;

import com.evolveum.midpoint.prism.PrismProperty;
import com.evolveum.midpoint.prism.impl.marshaller.ItemPathParserTemp;
import com.evolveum.midpoint.prism.path.UniformItemPath;
import com.evolveum.midpoint.xml.ns._public.common.common_3.ParamsType;
import com.evolveum.midpoint.xml.ns._public.common.common_3.TraceType;

import org.apache.commons.lang3.reflect.MethodUtils;
import org.apache.commons.text.StringSubstitutor;
import org.apache.commons.text.lookup.StringLookup;

import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.stream.Collectors;

import static com.evolveum.midpoint.schema.traces.TraceUtil.asStringList;
import static com.evolveum.midpoint.schema.traces.TraceUtil.selectByKey;

public class TemplateExpander {

public String expandTemplate(OpNode node, String nameTemplate) {
return new StringSubstitutor(createResolver(node), "${", "}", '\\')
.replace(nameTemplate);
}

private StringLookup createResolver(OpNode node) {
return spec -> {
String prefix;
String suffix;
String key;

String[] parts = spec.split(":");
if (parts.length == 1) {
prefix = "";
key = spec;
suffix = "";
} else if (parts.length == 2) {
prefix = parts[0];
key = parts[1];
suffix = "";
} else if (parts.length == 3) {
prefix = parts[0];
key = parts[1];
suffix = parts[2];
} else {
return "???";
}

List<String> values = new ArrayList<>();
if (prefix.isEmpty() || prefix.equals("p")) {
collectMatchingParams(values, key, node.getResult().getParams());
}
if (prefix.isEmpty() || prefix.equals("c")) {
collectMatchingParams(values, key, node.getResult().getContext());
}
if (prefix.isEmpty() || prefix.equals("r")) {
collectMatchingParams(values, key, node.getResult().getReturns());
}
if (prefix.isEmpty() || prefix.equals("t")) {
collectMatchingValues(values, key, node.getResult().getTrace());
}
if (prefix.equals("m")) {
collectFromMethod(values, key, node);
}
return String.join(", ", postprocess(values, suffix));
};
}

private void collectFromMethod(List<String> values, String key, OpNode node) {
try {
Object rv = MethodUtils.invokeExactMethod(node, key);
if (rv instanceof Collection) {
for (Object o : (Collection) rv) {
values.add(String.valueOf(o));
}
} else if (rv != null) {
values.add(String.valueOf(rv));
}
} catch (Throwable t) {
values.add("??? " + t.getMessage());
}
}

private void collectMatchingValues(List<String> values, String path, List<TraceType> traces) {
UniformItemPath itemPath = ItemPathParserTemp.parseFromString(path); // FIXME (hack)
for (TraceType trace : traces) {
PrismProperty property = trace.asPrismContainerValue().findProperty(itemPath);
if (property != null) {
for (Object realValue : property.getRealValues()) {
values.add(String.valueOf(realValue));
}
}
}
}

private void collectMatchingParams(List<String> values, String key, ParamsType params) {
int colon = key.indexOf(':');
String processing;
String realKey;
if (colon >= 0) {
realKey = key.substring(0, colon);
processing = key.substring(colon + 1);
} else {
realKey = key;
processing = "";
}
List<String> rawStringValues = asStringList(selectByKey(params, realKey));
values.addAll(postprocess(rawStringValues, processing));
}

private List<String> postprocess(List<String> raw, String processing) {
return raw.stream()
.map(s -> postprocess(s, processing))
.collect(Collectors.toList());
}

private String postprocess(String string, String processing) {
if (processing.contains("L")) {
string = string.toLowerCase();
}
return string;
}

}
Expand Up @@ -32,19 +32,26 @@ public class TraceParser {

@NotNull private final PrismContext prismContext;

@SuppressWarnings("WeakerAccess") // used externally
public TraceParser(@NotNull PrismContext prismContext) {
this.prismContext = prismContext;
}

public TracingOutputType parse(File file) throws IOException, SchemaException {
return parse(file, false);
}

public TracingOutputType parse(File file, boolean raw) throws IOException, SchemaException {
boolean isZip = file.getName().toLowerCase().endsWith(".zip");
return parse(new FileInputStream(file), isZip, file.getPath());
return parse(new FileInputStream(file), isZip, raw, file.getPath());
}

public TracingOutputType parse(InputStream inputStream, boolean isZip, String description) throws SchemaException, IOException {
return parse(inputStream, isZip, false, description);
}

public TracingOutputType parse(InputStream inputStream, boolean isZip, boolean raw, String description) throws SchemaException, IOException {
TracingOutputType wholeTracingOutput = getObject(inputStream, isZip, description);
if (wholeTracingOutput != null) {
if (!raw && wholeTracingOutput != null) {
new DictionaryExpander(wholeTracingOutput).expand();
new OperationCategorizer(wholeTracingOutput).categorize();
}
Expand Down

0 comments on commit 88bd8b2

Please sign in to comment.