From 029e910186d18f71af5cc7f50a260c2dc59a6a3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominik=20H=C3=B6lzl?= Date: Thu, 15 Mar 2018 17:58:39 +0100 Subject: [PATCH 1/7] hsmf: support writing properties, fix reading embedded message properties, support getting actual type of property value * Add support for writing MessagePropertiesChunk * MessagePropertiesChunk: Fix reading properties from embedded/attached MSG * PropertyValue: Add support for getting the actual property type and getting the raw bytes --- .../datatypes/ChunkBasedPropertyValue.java | 4 +- .../datatypes/MessagePropertiesChunk.java | 59 +++++-- .../poi/hsmf/datatypes/PropertiesChunk.java | 154 +++++++++++++++++- .../poi/hsmf/datatypes/PropertyValue.java | 35 +++- .../poi/hsmf/parsers/POIFSChunkParser.java | 3 +- 5 files changed, 225 insertions(+), 30 deletions(-) diff --git a/src/scratchpad/src/org/apache/poi/hsmf/datatypes/ChunkBasedPropertyValue.java b/src/scratchpad/src/org/apache/poi/hsmf/datatypes/ChunkBasedPropertyValue.java index 323065592f6..73b81814663 100644 --- a/src/scratchpad/src/org/apache/poi/hsmf/datatypes/ChunkBasedPropertyValue.java +++ b/src/scratchpad/src/org/apache/poi/hsmf/datatypes/ChunkBasedPropertyValue.java @@ -22,8 +22,8 @@ Licensed to the Apache Software Foundation (ASF) under one or more * TODO Provide a way to link these up with the chunks */ public class ChunkBasedPropertyValue extends PropertyValue { - public ChunkBasedPropertyValue(MAPIProperty property, long flags, byte[] offsetData) { - super(property, flags, offsetData); + public ChunkBasedPropertyValue(MAPIProperty property, long flags, byte[] offsetData, Types.MAPIType actualType) { + super(property, flags, offsetData, actualType); } @Override diff --git a/src/scratchpad/src/org/apache/poi/hsmf/datatypes/MessagePropertiesChunk.java b/src/scratchpad/src/org/apache/poi/hsmf/datatypes/MessagePropertiesChunk.java index ecc11a17675..5ba348ae035 100644 --- a/src/scratchpad/src/org/apache/poi/hsmf/datatypes/MessagePropertiesChunk.java +++ b/src/scratchpad/src/org/apache/poi/hsmf/datatypes/MessagePropertiesChunk.java @@ -20,6 +20,7 @@ Licensed to the Apache Software Foundation (ASF) under one or more import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.util.List; import org.apache.poi.util.LittleEndian; @@ -28,6 +29,7 @@ Licensed to the Apache Software Foundation (ASF) under one or more * byte header */ public class MessagePropertiesChunk extends PropertiesChunk { + private boolean isEmbedded; private long nextRecipientId; private long nextAttachmentId; private long recipientCount; @@ -37,6 +39,11 @@ public MessagePropertiesChunk(ChunkGroup parentGroup) { super(parentGroup); } + public MessagePropertiesChunk(ChunkGroup parentGroup, boolean isEmbedded) { + super(parentGroup); + this.isEmbedded = isEmbedded; + } + public long getNextRecipientId() { return nextRecipientId; } @@ -52,6 +59,22 @@ public long getRecipientCount() { public long getAttachmentCount() { return attachmentCount; } + + public void setNextRecipientId(long nextRecipientId) { + this.nextRecipientId = nextRecipientId; + } + + public void setNextAttachmentId(long nextAttachmentId) { + this.nextAttachmentId = nextAttachmentId; + } + + public void setRecipientCount(long recipientCount) { + this.recipientCount = recipientCount; + } + + public void setAttachmentCount(long attachmentCount) { + this.attachmentCount = attachmentCount; + } @Override public void readValue(InputStream stream) throws IOException { @@ -64,28 +87,34 @@ public void readValue(InputStream stream) throws IOException { recipientCount = LittleEndian.readUInt(stream); attachmentCount = LittleEndian.readUInt(stream); - // 8 bytes of reserved zeros - LittleEndian.readLong(stream); + if (!isEmbedded) { + // 8 bytes of reserved zeros (top level properties stream only) + LittleEndian.readLong(stream); + } // Now properties readProperties(stream); } @Override - public void writeValue(OutputStream out) throws IOException { - // 8 bytes of reserved zeros - out.write(new byte[8]); - - // Nexts and counts - LittleEndian.putUInt(nextRecipientId, out); - LittleEndian.putUInt(nextAttachmentId, out); - LittleEndian.putUInt(recipientCount, out); - LittleEndian.putUInt(attachmentCount, out); - - // 8 bytes of reserved zeros + protected List writeHeaderData(OutputStream out) throws IOException + { + // 8 bytes of reserved zeros + out.write(new byte[8]); + // Nexts and counts + LittleEndian.putUInt(nextRecipientId, out); + LittleEndian.putUInt(nextAttachmentId, out); + LittleEndian.putUInt(recipientCount, out); + LittleEndian.putUInt(attachmentCount, out); + // 8 bytes of reserved zeros + if (!isEmbedded) { out.write(new byte[8]); + } + // Now properties. + return super.writeHeaderData(out); + } - // Now properties - writeProperties(out); + @Override + public void writeValue(OutputStream out) throws IOException { } } diff --git a/src/scratchpad/src/org/apache/poi/hsmf/datatypes/PropertiesChunk.java b/src/scratchpad/src/org/apache/poi/hsmf/datatypes/PropertiesChunk.java index 0aa6162ac2a..c205ffadfac 100644 --- a/src/scratchpad/src/org/apache/poi/hsmf/datatypes/PropertiesChunk.java +++ b/src/scratchpad/src/org/apache/poi/hsmf/datatypes/PropertiesChunk.java @@ -17,13 +17,18 @@ Licensed to the Apache Software Foundation (ASF) under one or more package org.apache.poi.hsmf.datatypes; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; +import java.util.Map.Entry; import org.apache.poi.hsmf.datatypes.PropertyValue.BooleanPropertyValue; import org.apache.poi.hsmf.datatypes.PropertyValue.CurrencyPropertyValue; @@ -35,6 +40,7 @@ Licensed to the Apache Software Foundation (ASF) under one or more import org.apache.poi.hsmf.datatypes.PropertyValue.ShortPropertyValue; import org.apache.poi.hsmf.datatypes.PropertyValue.TimePropertyValue; import org.apache.poi.hsmf.datatypes.Types.MAPIType; +import org.apache.poi.poifs.filesystem.DirectoryEntry; import org.apache.poi.util.IOUtils; import org.apache.poi.util.LittleEndian; import org.apache.poi.util.LittleEndian.BufferUnderrunException; @@ -51,9 +57,16 @@ Licensed to the Apache Software Foundation (ASF) under one or more public abstract class PropertiesChunk extends Chunk { public static final String NAME = "__properties_version1.0"; - //arbitrarily selected; may need to increase + // arbitrarily selected; may need to increase private static final int MAX_RECORD_LENGTH = 1_000_000; + // standard prefix, defined in the spec + public static final String VARIABLE_LENGTH_PROPERTY_PREFIX = "__substg1.0_"; + + // standard property flags, defined in the spec + public static final int PROPERTIES_FLAG_READABLE = 2; + public static final int PROPERTIES_FLAG_WRITEABLE = 4; + /** For logging problems we spot with the file */ private POILogger logger = POILogFactory.getLogger(PropertiesChunk.class); @@ -105,6 +118,13 @@ public Map> getProperties() { return props; } + /** + * Defines a property. Multi-valued properties are not yet supported. + */ + public void setProperty(PropertyValue value) { + properties.put(value.getProperty(), value); + } + /** * Returns all values for the given property, looking up chunk based ones as * required, of null if none exist @@ -239,7 +259,7 @@ protected void readProperties(InputStream value) throws IOException { PropertyValue propVal = null; if (isPointer) { // We'll match up the chunk later - propVal = new ChunkBasedPropertyValue(prop, flags, data); + propVal = new ChunkBasedPropertyValue(prop, flags, data, type); } else if (type == Types.NULL) { propVal = new NullPropertyValue(prop, flags, data); } else if (type == Types.BOOLEAN) { @@ -261,7 +281,7 @@ protected void readProperties(InputStream value) throws IOException { } // TODO Add in the rest of the types else { - propVal = new PropertyValue(prop, flags, data); + propVal = new PropertyValue(prop, flags, data, type); } if (properties.get(prop) != null) { @@ -279,4 +299,132 @@ protected void readProperties(InputStream value) throws IOException { protected void writeProperties(OutputStream out) throws IOException { // TODO } + + /** + * Writes this chunk in the specified {@code DirectoryEntry}. + * + * @param directory + * The directory. + * @throws IOException + * If an I/O error occurs. + */ + public void writeTo(DirectoryEntry directory) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + List values = writeHeaderData(baos); + baos.close(); + + // write the header data with the properties declaration + directory.createDocument(org.apache.poi.hsmf.datatypes.PropertiesChunk.NAME, + new ByteArrayInputStream(baos.toByteArray())); + + // write the property values + writeNodeData(directory, values); + } + + /** + * Write the nodes for variable-length data. Those properties are returned by + * {@link #writeHeaderData(java.io.OutputStream)}. + * + * @param directory + * The directory. + * @param values + * The values. + * @throws IOException + * If an I/O error occurs. + */ + protected void writeNodeData(DirectoryEntry directory, List values) throws IOException { + for (PropertyValue value : values) { + byte[] bytes = value.getRawValue(); + String nodeName = VARIABLE_LENGTH_PROPERTY_PREFIX + getFileName(value.getProperty(), value.getActualType()); + directory.createDocument(nodeName, new ByteArrayInputStream(bytes)); + } + } + + /** + * Writes the header of the properties. + * + * @param out + * The {@code OutputStream}. + * @return The variable-length properties that need to be written in another + * node. + * @throws IOException + * If an I/O error occurs. + */ + protected List writeHeaderData(OutputStream out) throws IOException { + List variableLengthProperties = new ArrayList<>(); + for (Entry entry : properties.entrySet()) { + MAPIProperty property = entry.getKey(); + PropertyValue value = entry.getValue(); + if (value == null) { + continue; + } + if (property.id < 0) { + continue; + } + // generic header + // page 23, point 2.4.2 + // tag is the property id and its type + long tag = Long.parseLong(getFileName(property, value.getActualType()), 16); + LittleEndian.putUInt(tag, out); + LittleEndian.putUInt(value.getFlags(), out); // readable + writable + + MAPIType type = getTypeMapping(value.getActualType()); + if (type.isFixedLength()) { + // page 11, point 2.1.2 + writeFixedLengthValueHeader(out, property, type, value); + } else { + // page 12, point 2.1.3 + writeVariableLengthValueHeader(out, property, type, value); + variableLengthProperties.add(value); + } + } + return variableLengthProperties; + } + + private void writeFixedLengthValueHeader(OutputStream out, MAPIProperty property, MAPIType type, PropertyValue value) + throws IOException { + // fixed type header + // page 24, point 2.4.2.1.1 + byte[] bytes = value.getRawValue(); + int length = bytes != null ? bytes.length : 0; + if (bytes != null) { + // Little endian. + byte[] reversed = new byte[bytes.length]; + for (int i = 0; i < bytes.length; ++i) { + reversed[bytes.length - i - 1] = bytes[i]; + } + out.write(reversed); + } + out.write(new byte[8 - length]); + } + + private void writeVariableLengthValueHeader(OutputStream out, MAPIProperty propertyEx, MAPIType type, + PropertyValue value) throws IOException { + // variable length header + // page 24, point 2.4.2.2 + byte[] bytes = value.getRawValue(); + int length = bytes != null ? bytes.length : 0; + // alter the length, as specified in page 25 + if (type == Types.UNICODE_STRING) { + length += 2; + } else if (type == Types.ASCII_STRING) { + length += 1; + } + LittleEndian.putUInt(length, out); + // specified in page 25 + LittleEndian.putUInt(0, out); + } + + private String getFileName(MAPIProperty property, MAPIType actualType) { + String str = Integer.toHexString(property.id).toUpperCase(Locale.ROOT); + while (str.length() < 4) { + str = "0" + str; + } + MAPIType type = getTypeMapping(actualType); + return str + type.asFileEnding(); + } + + private MAPIType getTypeMapping(MAPIType type) { + return type == Types.ASCII_STRING ? Types.UNICODE_STRING : type; + } } diff --git a/src/scratchpad/src/org/apache/poi/hsmf/datatypes/PropertyValue.java b/src/scratchpad/src/org/apache/poi/hsmf/datatypes/PropertyValue.java index 4dde440620e..80f59288a18 100644 --- a/src/scratchpad/src/org/apache/poi/hsmf/datatypes/PropertyValue.java +++ b/src/scratchpad/src/org/apache/poi/hsmf/datatypes/PropertyValue.java @@ -22,6 +22,7 @@ Licensed to the Apache Software Foundation (ASF) under one or more import org.apache.poi.util.LittleEndian; import org.apache.poi.util.LocaleUtil; +import org.apache.poi.hsmf.datatypes.Types.MAPIType; /** * An instance of a {@link MAPIProperty} inside a {@link PropertiesChunk}. Where @@ -32,13 +33,21 @@ Licensed to the Apache Software Foundation (ASF) under one or more */ public class PropertyValue { private MAPIProperty property; + private MAPIType actualType; private long flags; protected byte[] data; public PropertyValue(MAPIProperty property, long flags, byte[] data) { + this.property = property; + this.flags = flags; + this.data = data; + this.actualType = property.usualType; + } + public PropertyValue(MAPIProperty property, long flags, byte[] data, MAPIType actualType) { this.property = property; this.flags = flags; this.data = data; + this.actualType = actualType; } public MAPIProperty getProperty() { @@ -56,6 +65,14 @@ public Object getValue() { return data; } + public byte[] getRawValue() { + return data; + } + + public MAPIType getActualType() { + return actualType; + } + public void setRawValue(byte[] value) { this.data = value; } @@ -78,7 +95,7 @@ public String toString() { public static class NullPropertyValue extends PropertyValue { public NullPropertyValue(MAPIProperty property, long flags, byte[] data) { - super(property, flags, data); + super(property, flags, data, org.apache.poi.hsmf.datatypes.Types.NULL); } @Override @@ -90,7 +107,7 @@ public Void getValue() { public static class BooleanPropertyValue extends PropertyValue { public BooleanPropertyValue(MAPIProperty property, long flags, byte[] data) { - super(property, flags, data); + super(property, flags, data, org.apache.poi.hsmf.datatypes.Types.BOOLEAN); } @Override @@ -112,7 +129,7 @@ public void setValue(boolean value) { public static class ShortPropertyValue extends PropertyValue { public ShortPropertyValue(MAPIProperty property, long flags, byte[] data) { - super(property, flags, data); + super(property, flags, data, org.apache.poi.hsmf.datatypes.Types.SHORT); } @Override @@ -130,7 +147,7 @@ public void setValue(short value) { public static class LongPropertyValue extends PropertyValue { public LongPropertyValue(MAPIProperty property, long flags, byte[] data) { - super(property, flags, data); + super(property, flags, data, org.apache.poi.hsmf.datatypes.Types.LONG); } @Override @@ -149,7 +166,7 @@ public void setValue(int value) { public static class LongLongPropertyValue extends PropertyValue { public LongLongPropertyValue(MAPIProperty property, long flags, byte[] data) { - super(property, flags, data); + super(property, flags, data, org.apache.poi.hsmf.datatypes.Types.LONG_LONG); } @Override @@ -168,7 +185,7 @@ public void setValue(long value) { public static class FloatPropertyValue extends PropertyValue { public FloatPropertyValue(MAPIProperty property, long flags, byte[] data) { - super(property, flags, data); + super(property, flags, data, org.apache.poi.hsmf.datatypes.Types.FLOAT); } @Override @@ -186,7 +203,7 @@ public void setValue(float value) { public static class DoublePropertyValue extends PropertyValue { public DoublePropertyValue(MAPIProperty property, long flags, byte[] data) { - super(property, flags, data); + super(property, flags, data, org.apache.poi.hsmf.datatypes.Types.DOUBLE); } @Override @@ -210,7 +227,7 @@ public static class CurrencyPropertyValue extends PropertyValue { private static final BigInteger SHIFT = BigInteger.valueOf(10000); public CurrencyPropertyValue(MAPIProperty property, long flags, byte[] data) { - super(property, flags, data); + super(property, flags, data, org.apache.poi.hsmf.datatypes.Types.CURRENCY); } @Override @@ -236,7 +253,7 @@ public static class TimePropertyValue extends PropertyValue { * (365L * 369L + 89L); public TimePropertyValue(MAPIProperty property, long flags, byte[] data) { - super(property, flags, data); + super(property, flags, data, org.apache.poi.hsmf.datatypes.Types.TIME); } @Override diff --git a/src/scratchpad/src/org/apache/poi/hsmf/parsers/POIFSChunkParser.java b/src/scratchpad/src/org/apache/poi/hsmf/parsers/POIFSChunkParser.java index e65a1216ddc..0d509ed708c 100644 --- a/src/scratchpad/src/org/apache/poi/hsmf/parsers/POIFSChunkParser.java +++ b/src/scratchpad/src/org/apache/poi/hsmf/parsers/POIFSChunkParser.java @@ -129,7 +129,8 @@ protected static void process(Entry entry, ChunkGroup grouping) { if (entryName.equals(PropertiesChunk.NAME)) { if (grouping instanceof Chunks) { // These should be the properties for the message itself - chunk = new MessagePropertiesChunk(grouping); + chunk = new MessagePropertiesChunk(grouping, + entry.getParent() != null && entry.getParent().getParent() != null); } else { // Will be properties on an attachment or recipient chunk = new StoragePropertiesChunk(grouping); From 99d7684352d5778d11dc3cb4c35326f2094c9624 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominik=20H=C3=B6lzl?= Date: Fri, 16 Mar 2018 11:21:48 +0100 Subject: [PATCH 2/7] adjusted formatting / no empty implementations --- .../datatypes/ChunkBasedPropertyValue.java | 3 + .../datatypes/MessagePropertiesChunk.java | 50 +++--- .../poi/hsmf/datatypes/PropertiesChunk.java | 169 +++++++++--------- .../poi/hsmf/datatypes/PropertyValue.java | 5 +- 4 files changed, 116 insertions(+), 111 deletions(-) diff --git a/src/scratchpad/src/org/apache/poi/hsmf/datatypes/ChunkBasedPropertyValue.java b/src/scratchpad/src/org/apache/poi/hsmf/datatypes/ChunkBasedPropertyValue.java index 73b81814663..ac3dfe75212 100644 --- a/src/scratchpad/src/org/apache/poi/hsmf/datatypes/ChunkBasedPropertyValue.java +++ b/src/scratchpad/src/org/apache/poi/hsmf/datatypes/ChunkBasedPropertyValue.java @@ -22,6 +22,9 @@ Licensed to the Apache Software Foundation (ASF) under one or more * TODO Provide a way to link these up with the chunks */ public class ChunkBasedPropertyValue extends PropertyValue { + public ChunkBasedPropertyValue(MAPIProperty property, long flags, byte[] offsetData) { + super(property, flags, offsetData); + } public ChunkBasedPropertyValue(MAPIProperty property, long flags, byte[] offsetData, Types.MAPIType actualType) { super(property, flags, offsetData, actualType); } diff --git a/src/scratchpad/src/org/apache/poi/hsmf/datatypes/MessagePropertiesChunk.java b/src/scratchpad/src/org/apache/poi/hsmf/datatypes/MessagePropertiesChunk.java index 5ba348ae035..7f72aa6d3c4 100644 --- a/src/scratchpad/src/org/apache/poi/hsmf/datatypes/MessagePropertiesChunk.java +++ b/src/scratchpad/src/org/apache/poi/hsmf/datatypes/MessagePropertiesChunk.java @@ -40,8 +40,8 @@ public MessagePropertiesChunk(ChunkGroup parentGroup) { } public MessagePropertiesChunk(ChunkGroup parentGroup, boolean isEmbedded) { - super(parentGroup); - this.isEmbedded = isEmbedded; + super(parentGroup); + this.isEmbedded = isEmbedded; } public long getNextRecipientId() { @@ -77,7 +77,7 @@ public void setAttachmentCount(long attachmentCount) { } @Override - public void readValue(InputStream stream) throws IOException { + protected void readProperties(InputStream stream) throws IOException { // 8 bytes of reserved zeros LittleEndian.readLong(stream); @@ -93,28 +93,38 @@ public void readValue(InputStream stream) throws IOException { } // Now properties - readProperties(stream); + super.readProperties(stream); } @Override - protected List writeHeaderData(OutputStream out) throws IOException - { - // 8 bytes of reserved zeros - out.write(new byte[8]); - // Nexts and counts - LittleEndian.putUInt(nextRecipientId, out); - LittleEndian.putUInt(nextAttachmentId, out); - LittleEndian.putUInt(recipientCount, out); - LittleEndian.putUInt(attachmentCount, out); - // 8 bytes of reserved zeros - if (!isEmbedded) { - out.write(new byte[8]); - } - // Now properties. - return super.writeHeaderData(out); + public void readValue(InputStream value) throws IOException { + readProperties(value); } @Override - public void writeValue(OutputStream out) throws IOException { + protected List writeProperties(OutputStream stream) throws IOException + { + // 8 bytes of reserved zeros + LittleEndian.putLong(0, stream); + + // Nexts and counts + LittleEndian.putUInt(nextRecipientId, stream); + LittleEndian.putUInt(nextAttachmentId, stream); + LittleEndian.putUInt(recipientCount, stream); + LittleEndian.putUInt(attachmentCount, stream); + + if (!isEmbedded) { + // 8 bytes of reserved zeros (top level properties stream only) + LittleEndian.putLong(0, stream); + } + + // Now properties. + return super.writeProperties(stream); + } + + @Override + public void writeValue(OutputStream stream) throws IOException { + // write properties without variable length properties + writeProperties(stream); } } diff --git a/src/scratchpad/src/org/apache/poi/hsmf/datatypes/PropertiesChunk.java b/src/scratchpad/src/org/apache/poi/hsmf/datatypes/PropertiesChunk.java index c205ffadfac..65479469290 100644 --- a/src/scratchpad/src/org/apache/poi/hsmf/datatypes/PropertiesChunk.java +++ b/src/scratchpad/src/org/apache/poi/hsmf/datatypes/PropertiesChunk.java @@ -122,7 +122,7 @@ public Map> getProperties() { * Defines a property. Multi-valued properties are not yet supported. */ public void setProperty(PropertyValue value) { - properties.put(value.getProperty(), value); + properties.put(value.getProperty(), value); } /** @@ -296,48 +296,44 @@ protected void readProperties(InputStream value) throws IOException { } } - protected void writeProperties(OutputStream out) throws IOException { - // TODO - } - /** * Writes this chunk in the specified {@code DirectoryEntry}. * * @param directory - * The directory. + * The directory. * @throws IOException - * If an I/O error occurs. + * If an I/O error occurs. */ - public void writeTo(DirectoryEntry directory) throws IOException { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - List values = writeHeaderData(baos); - baos.close(); + public void writeProperties(DirectoryEntry directory) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + List values = writeProperties(baos); + baos.close(); - // write the header data with the properties declaration - directory.createDocument(org.apache.poi.hsmf.datatypes.PropertiesChunk.NAME, - new ByteArrayInputStream(baos.toByteArray())); + // write the header data with the properties declaration + directory.createDocument(org.apache.poi.hsmf.datatypes.PropertiesChunk.NAME, + new ByteArrayInputStream(baos.toByteArray())); - // write the property values - writeNodeData(directory, values); + // write the property values + writeNodeData(directory, values); } /** * Write the nodes for variable-length data. Those properties are returned by - * {@link #writeHeaderData(java.io.OutputStream)}. + * {@link #writeProperties(java.io.OutputStream)}. * * @param directory - * The directory. + * The directory. * @param values - * The values. + * The values. * @throws IOException - * If an I/O error occurs. + * If an I/O error occurs. */ protected void writeNodeData(DirectoryEntry directory, List values) throws IOException { - for (PropertyValue value : values) { - byte[] bytes = value.getRawValue(); - String nodeName = VARIABLE_LENGTH_PROPERTY_PREFIX + getFileName(value.getProperty(), value.getActualType()); - directory.createDocument(nodeName, new ByteArrayInputStream(bytes)); - } + for (PropertyValue value : values) { + byte[] bytes = value.getRawValue(); + String nodeName = VARIABLE_LENGTH_PROPERTY_PREFIX + getFileName(value.getProperty(), value.getActualType()); + directory.createDocument(nodeName, new ByteArrayInputStream(bytes)); + } } /** @@ -350,81 +346,80 @@ protected void writeNodeData(DirectoryEntry directory, List value * @throws IOException * If an I/O error occurs. */ - protected List writeHeaderData(OutputStream out) throws IOException { - List variableLengthProperties = new ArrayList<>(); - for (Entry entry : properties.entrySet()) { - MAPIProperty property = entry.getKey(); - PropertyValue value = entry.getValue(); - if (value == null) { - continue; - } - if (property.id < 0) { - continue; - } - // generic header - // page 23, point 2.4.2 - // tag is the property id and its type - long tag = Long.parseLong(getFileName(property, value.getActualType()), 16); - LittleEndian.putUInt(tag, out); - LittleEndian.putUInt(value.getFlags(), out); // readable + writable - - MAPIType type = getTypeMapping(value.getActualType()); - if (type.isFixedLength()) { - // page 11, point 2.1.2 - writeFixedLengthValueHeader(out, property, type, value); - } else { - // page 12, point 2.1.3 - writeVariableLengthValueHeader(out, property, type, value); - variableLengthProperties.add(value); + protected List writeProperties(OutputStream out) throws IOException { + List variableLengthProperties = new ArrayList<>(); + for (Entry entry : properties.entrySet()) { + MAPIProperty property = entry.getKey(); + PropertyValue value = entry.getValue(); + if (value == null) { + continue; + } + if (property.id < 0) { + continue; + } + // generic header + // page 23, point 2.4.2 + // tag is the property id and its type + long tag = Long.parseLong(getFileName(property, value.getActualType()), 16); + LittleEndian.putUInt(tag, out); + LittleEndian.putUInt(value.getFlags(), out); // readable + writable + + MAPIType type = getTypeMapping(value.getActualType()); + if (type.isFixedLength()) { + // page 11, point 2.1.2 + writeFixedLengthValueHeader(out, property, type, value); + } else { + // page 12, point 2.1.3 + writeVariableLengthValueHeader(out, property, type, value); + variableLengthProperties.add(value); + } } - } - return variableLengthProperties; + return variableLengthProperties; } - private void writeFixedLengthValueHeader(OutputStream out, MAPIProperty property, MAPIType type, PropertyValue value) - throws IOException { - // fixed type header - // page 24, point 2.4.2.1.1 - byte[] bytes = value.getRawValue(); - int length = bytes != null ? bytes.length : 0; - if (bytes != null) { - // Little endian. - byte[] reversed = new byte[bytes.length]; - for (int i = 0; i < bytes.length; ++i) { - reversed[bytes.length - i - 1] = bytes[i]; + private void writeFixedLengthValueHeader(OutputStream out, MAPIProperty property, MAPIType type, PropertyValue value) throws IOException { + // fixed type header + // page 24, point 2.4.2.1.1 + byte[] bytes = value.getRawValue(); + int length = bytes != null ? bytes.length : 0; + if (bytes != null) { + // Little endian. + byte[] reversed = new byte[bytes.length]; + for (int i = 0; i < bytes.length; ++i) { + reversed[bytes.length - i - 1] = bytes[i]; + } + out.write(reversed); } - out.write(reversed); - } - out.write(new byte[8 - length]); + out.write(new byte[8 - length]); } private void writeVariableLengthValueHeader(OutputStream out, MAPIProperty propertyEx, MAPIType type, PropertyValue value) throws IOException { - // variable length header - // page 24, point 2.4.2.2 - byte[] bytes = value.getRawValue(); - int length = bytes != null ? bytes.length : 0; - // alter the length, as specified in page 25 - if (type == Types.UNICODE_STRING) { - length += 2; - } else if (type == Types.ASCII_STRING) { - length += 1; - } - LittleEndian.putUInt(length, out); - // specified in page 25 - LittleEndian.putUInt(0, out); + // variable length header + // page 24, point 2.4.2.2 + byte[] bytes = value.getRawValue(); + int length = bytes != null ? bytes.length : 0; + // alter the length, as specified in page 25 + if (type == Types.UNICODE_STRING) { + length += 2; + } else if (type == Types.ASCII_STRING) { + length += 1; + } + LittleEndian.putUInt(length, out); + // specified in page 25 + LittleEndian.putUInt(0, out); } private String getFileName(MAPIProperty property, MAPIType actualType) { - String str = Integer.toHexString(property.id).toUpperCase(Locale.ROOT); - while (str.length() < 4) { - str = "0" + str; - } - MAPIType type = getTypeMapping(actualType); - return str + type.asFileEnding(); + String str = Integer.toHexString(property.id).toUpperCase(Locale.ROOT); + while (str.length() < 4) { + str = "0" + str; + } + MAPIType type = getTypeMapping(actualType); + return str + type.asFileEnding(); } private MAPIType getTypeMapping(MAPIType type) { - return type == Types.ASCII_STRING ? Types.UNICODE_STRING : type; + return type == Types.ASCII_STRING ? Types.UNICODE_STRING : type; } } diff --git a/src/scratchpad/src/org/apache/poi/hsmf/datatypes/PropertyValue.java b/src/scratchpad/src/org/apache/poi/hsmf/datatypes/PropertyValue.java index 80f59288a18..1d809cba8db 100644 --- a/src/scratchpad/src/org/apache/poi/hsmf/datatypes/PropertyValue.java +++ b/src/scratchpad/src/org/apache/poi/hsmf/datatypes/PropertyValue.java @@ -38,10 +38,7 @@ public class PropertyValue { protected byte[] data; public PropertyValue(MAPIProperty property, long flags, byte[] data) { - this.property = property; - this.flags = flags; - this.data = data; - this.actualType = property.usualType; + this(property, flags, data, property.usualType); } public PropertyValue(MAPIProperty property, long flags, byte[] data, MAPIType actualType) { this.property = property; From e3ba7b9dea7d37aedcac38231334c623c1a68681 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominik=20H=C3=B6lzl?= Date: Fri, 16 Mar 2018 11:26:54 +0100 Subject: [PATCH 3/7] adjusted formatting --- .../org/apache/poi/hsmf/datatypes/MessagePropertiesChunk.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scratchpad/src/org/apache/poi/hsmf/datatypes/MessagePropertiesChunk.java b/src/scratchpad/src/org/apache/poi/hsmf/datatypes/MessagePropertiesChunk.java index 7f72aa6d3c4..327fa487674 100644 --- a/src/scratchpad/src/org/apache/poi/hsmf/datatypes/MessagePropertiesChunk.java +++ b/src/scratchpad/src/org/apache/poi/hsmf/datatypes/MessagePropertiesChunk.java @@ -122,7 +122,7 @@ protected List writeProperties(OutputStream stream) throws IOExce return super.writeProperties(stream); } - @Override + @Override public void writeValue(OutputStream stream) throws IOException { // write properties without variable length properties writeProperties(stream); From 7df242b7d486d877a8db3f6049caf63f7a27d5f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominik=20H=C3=B6lzl?= Date: Fri, 16 Mar 2018 15:34:29 +0100 Subject: [PATCH 4/7] add unit test for reading embedded msg properties and extracting embedded msg * test if fixed and variable length properties of embedded attached messages can be read * test if an embedded attached message can be extracted to a top level message and if re-parsing works --- .../poi/hsmf/TestExtractEmbeddedMSG.java | 177 ++++++++++++++++++ 1 file changed, 177 insertions(+) create mode 100644 src/scratchpad/testcases/org/apache/poi/hsmf/TestExtractEmbeddedMSG.java diff --git a/src/scratchpad/testcases/org/apache/poi/hsmf/TestExtractEmbeddedMSG.java b/src/scratchpad/testcases/org/apache/poi/hsmf/TestExtractEmbeddedMSG.java new file mode 100644 index 00000000000..d706630846c --- /dev/null +++ b/src/scratchpad/testcases/org/apache/poi/hsmf/TestExtractEmbeddedMSG.java @@ -0,0 +1,177 @@ +package org.apache.poi.hsmf; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Calendar; +import java.util.Map; + +import org.apache.poi.POIDataSamples; +import org.apache.poi.hsmf.datatypes.AttachmentChunks; +import org.apache.poi.hsmf.datatypes.Chunk; +import org.apache.poi.hsmf.datatypes.ChunkBasedPropertyValue; +import org.apache.poi.hsmf.datatypes.MAPIProperty; +import org.apache.poi.hsmf.datatypes.MessagePropertiesChunk; +import org.apache.poi.hsmf.datatypes.NameIdChunks; +import org.apache.poi.hsmf.datatypes.PropertiesChunk; +import org.apache.poi.hsmf.datatypes.PropertyValue; +import org.apache.poi.hsmf.datatypes.RecipientChunks; +import org.apache.poi.hsmf.datatypes.Types; +import org.apache.poi.hsmf.datatypes.Types.MAPIType; +import org.apache.poi.hsmf.exceptions.ChunkNotFoundException; +import org.apache.poi.poifs.filesystem.DirectoryEntry; +import org.apache.poi.poifs.filesystem.Entry; +import org.apache.poi.poifs.filesystem.EntryUtils; +import org.apache.poi.poifs.filesystem.POIFSFileSystem; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +public class TestExtractEmbeddedMSG { + private static MAPIMessage pdfMsgAttachments; + + /** + * Initialize this test, load up the attachment_msg_pdf.msg mapi message. + * + * @throws Exception + */ + @BeforeClass + public static void setUp() throws IOException { + POIDataSamples samples = POIDataSamples.getHSMFInstance(); + pdfMsgAttachments = new MAPIMessage(samples.openResourceAsStream("attachment_msg_pdf.msg")); + } + + @AfterClass + public static void tearDown() throws IOException { + pdfMsgAttachments.close(); + } + + /** + * Test to see if embedded message properties can be read, extracted, and + * re-parsed + * + * @throws ChunkNotFoundException + * + */ + @Test + public void testEmbeddedMSGProperties() throws IOException, ChunkNotFoundException { + AttachmentChunks[] attachments = pdfMsgAttachments.getAttachmentFiles(); + assertEquals(2, attachments.length); + if (attachments.length == 2) { + MAPIMessage attachedMsg = attachments[0].getEmbeddedMessage(); + assertNotNull(attachedMsg); + // test properties of embedded message + testFixedAndVariableLengthPropertiesOfAttachedMSG(attachedMsg); + // rebuild top level message from embedded message + try (POIFSFileSystem extractedAttachedMsg = rebuildFromAttached(attachedMsg)) { + try (ByteArrayOutputStream extractedAttachedMsgOut = new ByteArrayOutputStream()) { + extractedAttachedMsg.writeFilesystem(extractedAttachedMsgOut); + byte[] extratedAttachedMsgRaw = extractedAttachedMsgOut.toByteArray(); + MAPIMessage extractedMsgTopLevel = new MAPIMessage( + new ByteArrayInputStream(extratedAttachedMsgRaw)); + // test properties of rebuilt embedded message + testFixedAndVariableLengthPropertiesOfAttachedMSG(extractedMsgTopLevel); + } + } + } + } + + private void testFixedAndVariableLengthPropertiesOfAttachedMSG(MAPIMessage msg) throws ChunkNotFoundException { + // test fixed length property + msg.setReturnNullOnMissingChunk(true); + Calendar messageDate = msg.getMessageDate(); + assertNotNull(messageDate); + Calendar expectedMessageDate = Calendar.getInstance(); + expectedMessageDate.set(2010, 05, 18, 1, 52, 19); // 2010/06/18 01:52:19 + expectedMessageDate.set(Calendar.MILLISECOND, 0); + assertEquals(expectedMessageDate.getTimeInMillis(), messageDate.getTimeInMillis()); + // test variable length property + assertEquals(msg.getSubject(), "Test Attachment"); + } + + private POIFSFileSystem rebuildFromAttached(MAPIMessage attachedMsg) throws IOException { + // Create new MSG and copy properties. + POIFSFileSystem newDoc = new POIFSFileSystem(); + MessagePropertiesChunk topLevelChunk = new MessagePropertiesChunk(null); + // Copy attachments and recipients. + int recipientscount = 0; + int attachmentscount = 0; + for (Entry entry : attachedMsg.getDirectory()) { + if (entry.getName().startsWith(RecipientChunks.PREFIX)) { + recipientscount++; + DirectoryEntry newDir = newDoc.createDirectory(entry.getName()); + for (Entry e : ((DirectoryEntry) entry)) { + EntryUtils.copyNodeRecursively(e, newDir); + } + } else if (entry.getName().startsWith(AttachmentChunks.PREFIX)) { + attachmentscount++; + DirectoryEntry newDir = newDoc.createDirectory(entry.getName()); + for (Entry e : ((DirectoryEntry) entry)) { + EntryUtils.copyNodeRecursively(e, newDir); + } + } + } + // Copy properties from properties stream. + MessagePropertiesChunk mpc = attachedMsg.getMainChunks().getMessageProperties(); + for (Map.Entry p : mpc.getRawProperties().entrySet()) { + PropertyValue val = p.getValue(); + if (!(val instanceof ChunkBasedPropertyValue)) { + // Reverse data. + byte[] bytes = val.getRawValue(); + for (int idx = 0; idx < bytes.length / 2; idx++) { + byte xchg = bytes[bytes.length - 1 - idx]; + bytes[bytes.length - 1 - idx] = bytes[idx]; + bytes[idx] = xchg; + } + MAPIType type = val.getActualType(); + if (type != null && type != Types.UNKNOWN) { + topLevelChunk.setProperty(val); + } + } + } + // Create nameid entries. + DirectoryEntry nameid = newDoc.getRoot().createDirectory(NameIdChunks.NAME); + // GUID stream + nameid.createDocument(PropertiesChunk.DEFAULT_NAME_PREFIX + "00020102", new ByteArrayInputStream(new byte[0])); + // Entry stream + nameid.createDocument(PropertiesChunk.DEFAULT_NAME_PREFIX + "00030102", new ByteArrayInputStream(new byte[0])); + // String stream + nameid.createDocument(PropertiesChunk.DEFAULT_NAME_PREFIX + "00040102", new ByteArrayInputStream(new byte[0])); + // Base properties. + // Attachment/Recipient counter. + topLevelChunk.setAttachmentCount(attachmentscount); + topLevelChunk.setRecipientCount(recipientscount); + topLevelChunk.setNextAttachmentId(attachmentscount); + topLevelChunk.setNextRecipientId(recipientscount); + // Unicode string format. + topLevelChunk.setProperty(new PropertyValue(MAPIProperty.STORE_SUPPORT_MASK, + MessagePropertiesChunk.PROPERTIES_FLAG_READABLE | MessagePropertiesChunk.PROPERTIES_FLAG_WRITEABLE, + ByteBuffer.allocate(4).putInt(0x00040000).array())); + topLevelChunk.setProperty(new PropertyValue(MAPIProperty.HASATTACH, + MessagePropertiesChunk.PROPERTIES_FLAG_READABLE | MessagePropertiesChunk.PROPERTIES_FLAG_WRITEABLE, + attachmentscount == 0 ? new byte[] { 0 } : new byte[] { 1 })); + // Copy properties from MSG file system. + for (Chunk chunk : attachedMsg.getMainChunks().getChunks()) { + if (!(chunk instanceof MessagePropertiesChunk)) { + String entryName = chunk.getEntryName(); + String entryType = entryName.substring(entryName.length() - 4); + int iType = Integer.parseInt(entryType, 16); + MAPIType type = Types.getById(iType); + if (type != null && type != Types.UNKNOWN) { + MAPIProperty mprop = MAPIProperty.createCustom(chunk.getChunkId(), type, chunk.getEntryName()); + ByteArrayOutputStream data = new ByteArrayOutputStream(); + chunk.writeValue(data); + PropertyValue pval = new PropertyValue(mprop, MessagePropertiesChunk.PROPERTIES_FLAG_READABLE + | MessagePropertiesChunk.PROPERTIES_FLAG_WRITEABLE, data.toByteArray(), type); + topLevelChunk.setProperty(pval); + } + } + } + topLevelChunk.writeProperties(newDoc.getRoot()); + return newDoc; + } +} From 396acad6f43e0e8961a23c9c2d8f9ba36e837da9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominik=20H=C3=B6lzl?= Date: Fri, 16 Mar 2018 15:36:36 +0100 Subject: [PATCH 5/7] added unit test --- src/scratchpad/testcases/org/apache/poi/hsmf/AllHSMFTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/scratchpad/testcases/org/apache/poi/hsmf/AllHSMFTests.java b/src/scratchpad/testcases/org/apache/poi/hsmf/AllHSMFTests.java index fc042c747ce..77dfb5e4cb0 100644 --- a/src/scratchpad/testcases/org/apache/poi/hsmf/AllHSMFTests.java +++ b/src/scratchpad/testcases/org/apache/poi/hsmf/AllHSMFTests.java @@ -38,7 +38,8 @@ Licensed to the Apache Software Foundation (ASF) under one or more TestOutlookTextExtractor.class, TestPOIFSChunkParser.class, TestMessageSubmissionChunkY2KRead.class, - TestMessageSubmissionChunk.class + TestMessageSubmissionChunk.class, + TestExtractEmbeddedMSG.class }) public class AllHSMFTests { } From 8a32c03e9453d95f35eaced1d5f43f8028d161d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominik=20H=C3=B6lzl?= Date: Mon, 19 Mar 2018 08:57:10 +0100 Subject: [PATCH 6/7] fixed unit test fixed unit test --- .../testcases/org/apache/poi/hsmf/TestExtractEmbeddedMSG.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/scratchpad/testcases/org/apache/poi/hsmf/TestExtractEmbeddedMSG.java b/src/scratchpad/testcases/org/apache/poi/hsmf/TestExtractEmbeddedMSG.java index d706630846c..80d27e43b69 100644 --- a/src/scratchpad/testcases/org/apache/poi/hsmf/TestExtractEmbeddedMSG.java +++ b/src/scratchpad/testcases/org/apache/poi/hsmf/TestExtractEmbeddedMSG.java @@ -9,6 +9,7 @@ import java.nio.ByteBuffer; import java.util.Calendar; import java.util.Map; +import java.util.TimeZone; import org.apache.poi.POIDataSamples; import org.apache.poi.hsmf.datatypes.AttachmentChunks; @@ -86,7 +87,8 @@ private void testFixedAndVariableLengthPropertiesOfAttachedMSG(MAPIMessage msg) Calendar messageDate = msg.getMessageDate(); assertNotNull(messageDate); Calendar expectedMessageDate = Calendar.getInstance(); - expectedMessageDate.set(2010, 05, 18, 1, 52, 19); // 2010/06/18 01:52:19 + expectedMessageDate.set(2010, 05, 17, 23, 52, 19); // 2010/06/17 23:52:19 GMT + expectedMessageDate.setTimeZone(TimeZone.getTimeZone("GMT")); expectedMessageDate.set(Calendar.MILLISECOND, 0); assertEquals(expectedMessageDate.getTimeInMillis(), messageDate.getTimeInMillis()); // test variable length property From 1971131042cbb690a26f0b9219e1cda135b5e150 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominik=20H=C3=B6lzl?= Date: Mon, 19 Mar 2018 11:25:22 +0100 Subject: [PATCH 7/7] PropertiesChunk.writeFixedLengthValueHeader: PropertyValue.data is handled as little endian * PropertiesChunk.writeFixedLengthValueHeader: PropertyValue.data is handled as little endian (PropertyValue.data is already expected to hold data as little endian) --- .../apache/poi/hsmf/datatypes/PropertiesChunk.java | 7 +------ .../apache/poi/hsmf/TestExtractEmbeddedMSG.java | 14 +++++--------- 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/src/scratchpad/src/org/apache/poi/hsmf/datatypes/PropertiesChunk.java b/src/scratchpad/src/org/apache/poi/hsmf/datatypes/PropertiesChunk.java index 65479469290..56aa9bb93c3 100644 --- a/src/scratchpad/src/org/apache/poi/hsmf/datatypes/PropertiesChunk.java +++ b/src/scratchpad/src/org/apache/poi/hsmf/datatypes/PropertiesChunk.java @@ -383,12 +383,7 @@ private void writeFixedLengthValueHeader(OutputStream out, MAPIProperty property byte[] bytes = value.getRawValue(); int length = bytes != null ? bytes.length : 0; if (bytes != null) { - // Little endian. - byte[] reversed = new byte[bytes.length]; - for (int i = 0; i < bytes.length; ++i) { - reversed[bytes.length - i - 1] = bytes[i]; - } - out.write(reversed); + out.write(bytes); } out.write(new byte[8 - length]); } diff --git a/src/scratchpad/testcases/org/apache/poi/hsmf/TestExtractEmbeddedMSG.java b/src/scratchpad/testcases/org/apache/poi/hsmf/TestExtractEmbeddedMSG.java index 80d27e43b69..60350e8a236 100644 --- a/src/scratchpad/testcases/org/apache/poi/hsmf/TestExtractEmbeddedMSG.java +++ b/src/scratchpad/testcases/org/apache/poi/hsmf/TestExtractEmbeddedMSG.java @@ -122,13 +122,6 @@ private POIFSFileSystem rebuildFromAttached(MAPIMessage attachedMsg) throws IOEx for (Map.Entry p : mpc.getRawProperties().entrySet()) { PropertyValue val = p.getValue(); if (!(val instanceof ChunkBasedPropertyValue)) { - // Reverse data. - byte[] bytes = val.getRawValue(); - for (int idx = 0; idx < bytes.length / 2; idx++) { - byte xchg = bytes[bytes.length - 1 - idx]; - bytes[bytes.length - 1 - idx] = bytes[idx]; - bytes[idx] = xchg; - } MAPIType type = val.getActualType(); if (type != null && type != Types.UNKNOWN) { topLevelChunk.setProperty(val); @@ -150,9 +143,12 @@ private POIFSFileSystem rebuildFromAttached(MAPIMessage attachedMsg) throws IOEx topLevelChunk.setNextAttachmentId(attachmentscount); topLevelChunk.setNextRecipientId(recipientscount); // Unicode string format. - topLevelChunk.setProperty(new PropertyValue(MAPIProperty.STORE_SUPPORT_MASK, + byte[] storeSupportMaskData = new byte[4]; + PropertyValue.LongPropertyValue storeSupportPropertyValue = new PropertyValue.LongPropertyValue(MAPIProperty.STORE_SUPPORT_MASK, MessagePropertiesChunk.PROPERTIES_FLAG_READABLE | MessagePropertiesChunk.PROPERTIES_FLAG_WRITEABLE, - ByteBuffer.allocate(4).putInt(0x00040000).array())); + storeSupportMaskData); + storeSupportPropertyValue.setValue(0x00040000); + topLevelChunk.setProperty(storeSupportPropertyValue); topLevelChunk.setProperty(new PropertyValue(MAPIProperty.HASATTACH, MessagePropertiesChunk.PROPERTIES_FLAG_READABLE | MessagePropertiesChunk.PROPERTIES_FLAG_WRITEABLE, attachmentscount == 0 ? new byte[] { 0 } : new byte[] { 1 }));