Skip to content

Commit

Permalink
Compress traces using object dictionary
Browse files Browse the repository at this point in the history
  • Loading branch information
@mederly
mederly committed Jul 13, 2019
1 parent 6771e48 commit 01cd396
Show file tree
Hide file tree
Showing 5 changed files with 247 additions and 16 deletions.
14 changes: 2 additions & 12 deletions infra/common/src/main/java/com/evolveum/midpoint/common/crypto/CryptoUtil.java
View file
Expand Up @@ -258,18 +258,8 @@ public void visit(JaxbVisitable visitable) {
} catch (EncryptionException e) {
throw new TunnelException(e);
}
} else if (visitable instanceof Containerable) {
((Containerable) visitable).asPrismContainerValue().accept(this);
} else if (visitable instanceof Referencable) {
PrismObject<?> object = ((Referencable) visitable).asReferenceValue().getObject();
if (object != null) {
object.accept(this);
}
} else if (visitable instanceof RawType) {
RawType raw = (RawType) visitable;
if (raw.isParsed()) {
raw.getAlreadyParsedValue().accept(this);
}
} else {
JaxbVisitable.visitPrismStructure(visitable, this);
}
}

Expand Down
17 changes: 17 additions & 0 deletions infra/prism-api/src/main/java/com/evolveum/midpoint/prism/JaxbVisitable.java
View file
Expand Up @@ -15,6 +15,8 @@
*/
package com.evolveum.midpoint.prism;

import com.evolveum.prism.xml.ns._public.types_3.RawType;

/**
* Represents visitable JAXB bean.
*
Expand All @@ -25,4 +27,19 @@ public interface JaxbVisitable {

void accept(JaxbVisitor visitor);

static void visitPrismStructure(JaxbVisitable visitable, Visitor prismVisitor) {
if (visitable instanceof Containerable) {
((Containerable) visitable).asPrismContainerValue().accept(prismVisitor);
} else if (visitable instanceof Referencable) {
PrismObject<?> object = ((Referencable) visitable).asReferenceValue().getObject();
if (object != null) {
object.accept(prismVisitor);
}
} else if (visitable instanceof RawType) {
RawType raw = (RawType) visitable;
if (raw.isParsed()) {
raw.getAlreadyParsedValue().accept(prismVisitor);
}
}
}
}
1 change: 1 addition & 0 deletions infra/schema/src/main/java/com/evolveum/midpoint/schema/constants/SchemaConstants.java
View file
Expand Up @@ -629,4 +629,5 @@ public abstract class SchemaConstants {
//enum defs
public final static QName D_LOGGING_LEVEL_TYPE = new QName(SchemaConstantsGenerated.NS_COMMON, "LoggingLevelType");

public static final String TRACE_DICTIONARY_PREFIX = "#dictionary#";
}
137 changes: 137 additions & 0 deletions infra/schema/src/main/resources/xml/ns/public/common/common-model-context-3.xsd
View file
Expand Up @@ -1829,4 +1829,141 @@
</xsd:restriction>
</xsd:simpleType>

<xsd:complexType name="TraceDictionaryType">
<xsd:annotation>
<xsd:documentation>
Object dictionary. Very experimental.
</xsd:documentation>
<xsd:appinfo>
<a:experimental>true</a:experimental>
<a:container>true</a:container>
<a:since>4.0</a:since>
</xsd:appinfo>
</xsd:annotation>
<xsd:sequence>
<xsd:element name="entry" type="tns:TraceDictionaryEntryType" minOccurs="0" maxOccurs="unbounded">
<xsd:annotation>
<xsd:documentation>
Single entry.
</xsd:documentation>
</xsd:annotation>
</xsd:element>
</xsd:sequence>
<xsd:attribute name="id" type="xsd:long"/>
</xsd:complexType>
<xsd:element name="traceDictionary" type="tns:TraceDictionaryType" />

<xsd:complexType name="TraceDictionaryEntryType">
<xsd:annotation>
<xsd:documentation>
One object.
</xsd:documentation>
<xsd:appinfo>
<a:experimental>true</a:experimental>
<a:container>true</a:container>
<a:since>4.0</a:since>
</xsd:appinfo>
</xsd:annotation>
<xsd:sequence>
<xsd:element name="identifier" type="xsd:long" minOccurs="0">
<xsd:annotation>
<xsd:documentation>
Entry identifier.
</xsd:documentation>
<xsd:appinfo>
<a:composite>true</a:composite>
</xsd:appinfo>
</xsd:annotation>
</xsd:element>
<xsd:element name="object" type="tns:ObjectReferenceType" minOccurs="0">
<xsd:annotation>
<xsd:documentation>
Object. The reference must contain full object.
</xsd:documentation>
<xsd:appinfo>
<a:composite>true</a:composite>
</xsd:appinfo>
</xsd:annotation>
</xsd:element>
</xsd:sequence>
<xsd:attribute name="id" type="xsd:long"/>
</xsd:complexType>
<xsd:element name="traceDictionaryEntry" type="tns:TraceDictionaryEntryType" />

<xsd:complexType name="TracingOutputType">
<xsd:annotation>
<xsd:documentation>
Output of the tracing.
</xsd:documentation>
<xsd:appinfo>
<a:experimental>true</a:experimental>
<a:container>true</a:container>
<a:since>4.0</a:since>
</xsd:appinfo>
</xsd:annotation>
<xsd:sequence>
<xsd:element name="metadata" type="tns:TracingOutputMetadataType" minOccurs="0">
<xsd:annotation>
<xsd:documentation>
Metadata e.g. time of creation, etc.
</xsd:documentation>
</xsd:annotation>
</xsd:element>
<xsd:element name="result" type="tns:OperationResultType" minOccurs="0">
<xsd:annotation>
<xsd:documentation>
Operation result tree.
</xsd:documentation>
</xsd:annotation>
</xsd:element>
<xsd:element name="dictionary" type="tns:TraceDictionaryType" minOccurs="0">
<xsd:annotation>
<xsd:documentation>
Object dictionary.
</xsd:documentation>
</xsd:annotation>
</xsd:element>
</xsd:sequence>
<xsd:attribute name="id" type="xsd:long"/>
</xsd:complexType>
<xsd:element name="tracingOutput" type="tns:TracingOutputType" />

<xsd:complexType name="TracingOutputMetadataType">
<xsd:annotation>
<xsd:documentation>
Metadata e.g. time of creation, etc.
</xsd:documentation>
<xsd:appinfo>
<a:experimental>true</a:experimental>
<a:container>true</a:container>
<a:since>4.0</a:since>
</xsd:appinfo>
</xsd:annotation>
<xsd:sequence>
<xsd:element name="createTimestamp" type="xsd:dateTime" minOccurs="0">
<xsd:annotation>
<xsd:documentation>
When the tracing output was created.
</xsd:documentation>
</xsd:annotation>
</xsd:element>
<xsd:element name="creatorRef" type="tns:ObjectReferenceType" minOccurs="0">
<xsd:annotation>
<xsd:documentation>
The creator.
</xsd:documentation>
</xsd:annotation>
</xsd:element>
<xsd:element name="profile" type="tns:TracingProfileType" minOccurs="0">
<xsd:annotation>
<xsd:documentation>
Tracing profile used.
</xsd:documentation>
</xsd:annotation>
</xsd:element>
</xsd:sequence>
<xsd:attribute name="id" type="xsd:long"/>
</xsd:complexType>
<xsd:element name="tracingOutputMetadata" type="tns:TracingOutputMetadataType" />

</xsd:schema>
94 changes: 90 additions & 4 deletions ...k-quartz-impl/src/main/java/com/evolveum/midpoint/task/quartzimpl/tracing/TracerImpl.java
View file
Expand Up @@ -16,10 +16,13 @@

package com.evolveum.midpoint.task.quartzimpl.tracing;

import com.evolveum.midpoint.prism.PrismContext;
import com.evolveum.midpoint.prism.*;
import com.evolveum.midpoint.prism.equivalence.EquivalenceStrategy;
import com.evolveum.midpoint.prism.xml.XmlTypeConverter;
import com.evolveum.midpoint.repo.api.RepositoryService;
import com.evolveum.midpoint.repo.api.SystemConfigurationChangeDispatcher;
import com.evolveum.midpoint.repo.api.SystemConfigurationChangeListener;
import com.evolveum.midpoint.schema.constants.SchemaConstants;
import com.evolveum.midpoint.schema.result.CompiledTracingProfile;
import com.evolveum.midpoint.schema.result.OperationResult;
import com.evolveum.midpoint.schema.util.ObjectTypeUtil;
Expand Down Expand Up @@ -50,6 +53,7 @@
import java.nio.charset.StandardCharsets;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.Objects;
import java.util.stream.Collectors;

/**
Expand Down Expand Up @@ -99,8 +103,8 @@ public void storeTrace(Task task, OperationResult result) {
Map<String, String> templateParameters = createTemplateParameters(task, result); // todo evaluate lazily if needed
File file = createFileName(zip, tracingProfile, templateParameters);
try {
OperationResultType resultBean = result.createOperationResultType();
String xml = prismContext.xmlSerializer().serializeRealValue(resultBean);
TracingOutputType tracingOutput = createTracingOutput(result);
String xml = prismContext.xmlSerializer().serializeRealValue(tracingOutput);
if (zip) {
MiscUtil.writeZipFile(file, ZIP_ENTRY_NAME, xml, StandardCharsets.UTF_8);
LOGGER.info("Trace was written to {} ({} chars uncompressed)", file, xml.length());
Expand All @@ -124,6 +128,89 @@ public void storeTrace(Task task, OperationResult result) {
}
}

private TracingOutputType createTracingOutput(OperationResult result) {
TracingOutputType output = new TracingOutputType(prismContext);
output.beginMetadata()
.createTimestamp(XmlTypeConverter.createXMLGregorianCalendar(System.currentTimeMillis()))
.profile(result.getTracingProfile().getDefinition());
OperationResultType resultBean = result.createOperationResultType();
output.setDictionary(extractDictionary(resultBean));
output.setResult(resultBean);
return output;
}

// Extracting from JAXB objects currently does not work because RawType.getValue fails for
// raw versions of ObjectReferenceType holding full object
private static class ExtractingVisitor implements Visitor {

private final TraceDictionaryType dictionary;
private final PrismContext prismContext;

private ExtractingVisitor(TraceDictionaryType dictionary, PrismContext prismContext) {
this.dictionary = dictionary;
this.prismContext = prismContext;
}

// @Override
// public void visit(JaxbVisitable visitable) {
// JaxbVisitable.visitPrismStructure(visitable, this);
// }

@Override
public void visit(Visitable visitable) {
// if (visitable instanceof PrismPropertyValue) {
// PrismPropertyValue<?> pval = ((PrismPropertyValue) visitable);
// Object realValue = pval.getRealValue();
// if (realValue instanceof JaxbVisitable) {
// ((JaxbVisitable) realValue).accept(this);
// }
// } else
if (visitable instanceof PrismReferenceValue) {
PrismReferenceValue refVal = (PrismReferenceValue) visitable;
//noinspection unchecked
PrismObject<? extends ObjectType> object = refVal.getObject();
if (object != null) {
long entryId = findOrCreateEntry(object);
refVal.setObject(null);
refVal.setOid(SchemaConstants.TRACE_DICTIONARY_PREFIX + entryId);
}
}
}

private long findOrCreateEntry(PrismObject<? extends ObjectType> object) {
long max = 0;
for (TraceDictionaryEntryType entry : dictionary.getEntry()) {
PrismObject dictionaryObject = entry.getObject().asReferenceValue().getObject();
if (Objects.equals(object.getOid(), dictionaryObject.getOid()) && // todo remove if POV implements this
Objects.equals(object.getVersion(), dictionaryObject.getVersion()) && // todo remove if POV implements this
object.equals(dictionaryObject, EquivalenceStrategy.LITERAL)) {
return entry.getIdentifier();
}
if (entry.getIdentifier() > max) {
max = entry.getIdentifier();
}
}
long newId = max + 1;
System.out.println("Inserting object as entry #" + newId + ": " + object);
dictionary.beginEntry()
.identifier(newId)
.object(ObjectTypeUtil.createObjectRefWithFullObject(object, prismContext));
return newId;
}
}

private TraceDictionaryType extractDictionary(OperationResultType resultBean) {
TraceDictionaryType dictionary = new TraceDictionaryType(prismContext);
ExtractingVisitor extractingVisitor = new ExtractingVisitor(dictionary, prismContext);
extractDictionary(resultBean, extractingVisitor);
return dictionary;
}

private void extractDictionary(OperationResultType resultBean, ExtractingVisitor extractingVisitor) {
resultBean.getTrace().forEach(trace -> trace.asPrismContainerValue().accept(extractingVisitor));
resultBean.getPartialResults().forEach(partialResult -> extractDictionary(partialResult, extractingVisitor));
}

private Map<String, String> createTemplateParameters(Task task, OperationResult result) {
Map<String, String> rv = new HashMap<>();
SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss-SSS");
Expand Down Expand Up @@ -308,5 +395,4 @@ private TracingConfigurationType getTracingConfiguration() {
systemConfiguration.getInternals().getTracing() :
null;
}

}

0 comments on commit 01cd396

Please sign in to comment.