HSMF enhancements

src/scratchpad/src/org/apache/poi/hsmf/datatypes/AttachmentChunks.java

@@ -42,7 +42,7 @@ public class AttachmentChunks implements ChunkGroup {
     private static final POILogger LOG = POILogFactory.getLogger(AttachmentChunks.class);
     public static final String PREFIX = "__attach_version1.0_#";
 
-    private ByteChunk attachData;
+    private ByteStreamChunk attachData;
     private StringChunk attachExtension;
     private StringChunk attachFileName;
     private StringChunk attachLongFileName;
@@ -90,7 +90,7 @@ public MAPIMessage getEmbeddedMessage() throws IOException {
      * Returns the embedded object, if the attachment is an object based
      * embedding (image, document etc), or null if it's an embedded message
      */
-    public byte[] getEmbeddedAttachmentObject() {
+    public byte[] getEmbeddedAttachmentObject() throws IOException {
         if (attachData != null) {
             return attachData.getValue();
         }
@@ -113,7 +113,7 @@ public String getPOIFSName() {
     /**
      * @return the ATTACH_DATA chunk
      */
-    public ByteChunk getAttachData() {
+    public ByteStreamChunk getAttachData() {
         return attachData;
     }
 
@@ -182,8 +182,8 @@ public void record(Chunk chunk) {
         // - ATTACH_SIZE
         final int chunkId = chunk.getChunkId();
         if (chunkId == ATTACH_DATA.id) {
-            if (chunk instanceof ByteChunk) {
-                attachData = (ByteChunk) chunk;
+            if (chunk instanceof ByteStreamChunk) {
+                attachData = (ByteStreamChunk) chunk;
             } else if (chunk instanceof DirectoryChunk) {
                 attachmentDirectory = (DirectoryChunk) chunk;
             } else {

src/scratchpad/src/org/apache/poi/hsmf/datatypes/ByteStreamChunk.java

@@ -0,0 +1,157 @@
+/* ====================================================================
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+==================================================================== */
+
+package org.apache.poi.hsmf.datatypes;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+import org.apache.poi.util.IOUtils;
+
+import org.apache.poi.hsmf.datatypes.Types.MAPIType;
+
+/**
+ * A Chunk that holds a stream to binary data, normally unparsed. Generally as we know how
+ * to make sense of the contents, we create a new Chunk class and add a special
+ * case in the parser for them.
+ */
+
+public class ByteStreamChunk extends Chunk implements AutoCloseable {
+    private InputStream value;
+    boolean streamclosed;
+    byte[] bytevalue;
+
+    /**
+     * Creates a Byte Stream Chunk.
+     */
+    public ByteStreamChunk(String namePrefix, int chunkId, MAPIType type) {
+        super(namePrefix, chunkId, type);
+    }
+
+    /**
+     * Create a Byte Stream Chunk, with the specified type.
+     */
+    public ByteStreamChunk(int chunkId, MAPIType type) {
+        super(chunkId, type);
+    }
+
+    @Override
+    public void readValue(InputStream value) throws IOException {
+        this.value = value;
+    }
+
+    @Override
+    public void writeValue(OutputStream out) throws IOException {
+        if (bytevalue != null || streamclosed || value == null) {
+            if (bytevalue != null) {
+                out.write(bytevalue);
+            }
+        }
+        else {
+            value.reset();
+            IOUtils.copy(value, out);
+        }
+    }
+
+    /**
+     * Get bytes directly.
+     */
+    public byte[] getValue() throws IOException {
+        if (value != null && bytevalue == null && !streamclosed) {
+            try (InputStream valueres = value) {
+                valueres.reset();
+                bytevalue = IOUtils.toByteArray(valueres);
+            } finally {
+                streamclosed = true;
+                value = null;
+            }
+        }
+        return bytevalue;
+    }
+
+    /**
+     * Set bytes directly.
+     */
+    public void setValue(byte[] value) {
+        this.bytevalue = value;
+        this.value = new ByteArrayInputStream(value);
+    }
+
+    /**
+     * Closes the underlying stream.
+     */
+    public void close() throws IOException {
+        if (value != null) {
+            value.close();
+        }
+    }
+
+    /**
+     * Returns the data in a debug-friendly string format
+     */
+    @Override
+    public String toString() {
+        try {
+            return toDebugFriendlyString(getValue());
+        } catch (Exception e) {
+            return e.toString();
+        }
+    }
+
+    /**
+     * Formats the byte array in a debug-friendly way, showing all of a short
+     * array, and the start of a longer one.
+     */
+    protected static String toDebugFriendlyString(byte[] value) {
+        if (value == null) {
+            return "(Null Byte Array)";
+        }
+
+        StringBuffer text = new StringBuffer();
+        text.append("Bytes len=").append(value.length);
+        text.append(" [");
+
+        int limit = Math.min(value.length, 16);
+        if (value.length > 16) {
+            limit = 12;
+        }
+        for (int i = 0; i < limit; i++) {
+            if (i > 0) {
+                text.append(',');
+            }
+            text.append(value[i]);
+        }
+        if (value.length > 16) {
+            text.append(",....");
+        }
+        text.append("]");
+        return text.toString();
+    }
+
+    /**
+     * Returns the data, formatted as a string assuming it was a non-unicode
+     * string. If your data isn't in fact stored as basically ASCII, don't
+     * expect this to return much of any sense....
+     * 
+     * @return the data formatted as a string
+     */
+    public String getAs7bitString() throws IOException {
+        return StringChunk.parseAs7BitData(getValue());
+    }
+}

src/scratchpad/src/org/apache/poi/hsmf/datatypes/Chunks.java

@@ -45,7 +45,13 @@ public final class Chunks implements ChunkGroupWithProperties {
      * (variable size), but in some cases (eg Unknown) you may get more.
      */
     private Map<MAPIProperty, List<Chunk>> allChunks = new HashMap<>();
-
+
+    /**
+     * Holds all the unknown properties that were found, indexed by their property id and property type.
+     * All unknown properties have a custom properties instance.
+     */
+    private Map<Long, MAPIProperty> unknownProperties = new HashMap<>();
+
     /** Type of message that the MSG represents (ie. IPM.Note) */
     private StringChunk messageClass;
     /** BODY Chunk, for plain/text messages */
@@ -188,7 +194,15 @@ public MessagePropertiesChunk getMessageProperties() {
     public void record(Chunk chunk) {
         // Work out what MAPIProperty this corresponds to
         MAPIProperty prop = MAPIProperty.get(chunk.getChunkId());
-
+        if (prop == MAPIProperty.UNKNOWN) {
+            long id = (chunk.getChunkId() << 16) + chunk.getType().getId();
+            prop = unknownProperties.get(id);
+            if (prop == null) {
+                prop = MAPIProperty.createCustom(chunk.getChunkId(), chunk.getType(), chunk.getEntryName());
+                unknownProperties.put(id, prop);
+            }
+        }
+
         // Assign it for easy lookup, as best we can
         if (prop == MAPIProperty.MESSAGE_CLASS) {
             messageClass = (StringChunk) chunk;

src/scratchpad/src/org/apache/poi/hsmf/datatypes/MAPIProperty.java

@@ -790,6 +790,8 @@ public class MAPIProperty {
         new MAPIProperty(0x3f, BINARY, "ReceivedByEntryId", "PR_RECEIVED_BY_ENTRYID");
     public static final MAPIProperty RECEIVED_BY_NAME =
         new MAPIProperty(0x40, ASCII_STRING, "ReceivedByName", "PR_RECEIVED_BY_NAME");
+    public static final MAPIProperty RECEIVED_BY_SMTP_ADDRESS =
+        new MAPIProperty(0x5D07, ASCII_STRING, "ReceivedBySmtpAddress", "PR_RECEIVED_BY_SMTP_ADDRESS");
     public static final MAPIProperty RECIPIENT_DISPLAY_NAME =
         new MAPIProperty(0x5ff6, Types.UNICODE_STRING, "RecipientDisplayName", null);
     public static final MAPIProperty RECIPIENT_ENTRY_ID =