TIKA-2900: Add ability to exclude comments in Word extraction

tika-parsers/src/main/java/org/apache/tika/parser/microsoft/OfficeParserConfig.java

@@ -31,6 +31,7 @@ public class OfficeParserConfig implements Serializable {
     private boolean includeSlideNotes = true;
     private boolean includeSlideMasterContent = true;
     private boolean concatenatePhoneticRuns = true;
+    private boolean includeComments = true;
 
     private boolean useSAXDocxExtractor = false;
     private boolean useSAXPptxExtractor = false;
@@ -244,6 +245,26 @@ public boolean getIncludeSlideMasterContent() {
     public void setIncludeSlideMasterContent(boolean includeSlideMasterContent) {
         this.includeSlideMasterContent = includeSlideMasterContent;
     }
+
+    /**
+     * @since 1.23
+     * @return whether or not to process comments in a doc or docx file.
+     */
+    public boolean getIncludeComments() {
+        return includeComments;
+    }
+
+    /**
+     * Whether or not to include content from comments in a .doc or docx file.
+     * If set to <code>false</code>, the parser will not extract
+     * comments on paragraph runs.
+     *
+     * @since 1.23
+     * @param includeComments
+     */
+    public void setIncludeComments(boolean includeComments) {
+        this.includeComments = includeComments;
+    }
 }
 
 

tika-parsers/src/main/java/org/apache/tika/parser/microsoft/WordExtractor.java

@@ -198,8 +198,10 @@ protected void parse(
             xhtml.element("p", paragraph);
         }
 
-        for (String paragraph : wordExtractor.getCommentsText()) {
-            xhtml.element("p", paragraph);
+        if (officeParserConfig.getIncludeComments()) {
+            for (String paragraph : wordExtractor.getCommentsText()) {
+                xhtml.element("p", paragraph);
+            }
         }
 
         for (String paragraph : wordExtractor.getEndnoteText()) {

tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/OOXMLExtractorFactory.java

@@ -157,7 +157,7 @@ public static void parse(
             // We can rely on this being non-null.
             OfficeParserConfig config = context.get(OfficeParserConfig.class);
             if (config.getUseSAXDocxExtractor()) {
-                poiExtractor = trySXWPF(pkg);
+                poiExtractor = trySXWPF(pkg, config);
             }
             if (poiExtractor == null) {
                 poiExtractor = tryXSLF(pkg, config.getUseSAXPptxExtractor());
@@ -240,7 +240,7 @@ public static void parse(
         }
     }
 
-    private static POIXMLTextExtractor trySXWPF(OPCPackage pkg) throws XmlException, OpenXML4JException, IOException {
+    private static POIXMLTextExtractor trySXWPF(OPCPackage pkg, OfficeParserConfig config) throws XmlException, OpenXML4JException, IOException {
         PackageRelationshipCollection packageRelationshipCollection = pkg.getRelationshipsByType("http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument");
         if (packageRelationshipCollection.size() == 0) {
             packageRelationshipCollection = pkg.getRelationshipsByType("http://purl.oclc.org/ooxml/officeDocument/relationships/officeDocument");
@@ -253,7 +253,7 @@ private static POIXMLTextExtractor trySXWPF(OPCPackage pkg) throws XmlException,
         String targetContentType = corePart.getContentType();
         for (XWPFRelation relation : XWPFWordExtractor.SUPPORTED_TYPES) {
             if (targetContentType.equals(relation.getContentType())) {
-                return new XWPFEventBasedWordExtractor(pkg);
+                return new XWPFEventBasedWordExtractor(pkg, config);
             }
         }
         return null;

tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/SXWPFWordExtractorDecorator.java

@@ -167,10 +167,15 @@ private void handleDocumentPart(PackagePart documentPart, XHTMLContentHandler xh
                 XWPFRelation.ENDNOTE.getRelation(),
         }) {
             //skip footers if we shouldn't extract them
-            if (! config.getIncludeHeadersAndFooters() &&
+            if (!config.getIncludeHeadersAndFooters() &&
                     rel.equals(XWPFRelation.FOOTER.getRelation())) {
                 continue;
             }
+            //skip comments if we shouldn't extract them
+            if (!config.getIncludeComments() &&
+                    rel.equals(XWPFRelation.COMMENT.getRelation())) {
+                continue;
+            }
             try {
                 PackageRelationshipCollection prc = documentPart.getRelationshipsByType(rel);
                 if (prc != null) {

tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/XWPFWordExtractorDecorator.java

@@ -315,12 +315,13 @@ private void extractParagraph(XWPFParagraph paragraph, XWPFListManager listManag
             xhtml.endElement("a");
         }
 
-
-        // Now do any comments for the paragraph
-        XWPFCommentsDecorator comments = new XWPFCommentsDecorator(paragraph, null);
-        String commentText = comments.getCommentText();
-        if (commentText != null && commentText.length() > 0) {
-            xhtml.characters(commentText);
+        if (config.getIncludeComments()) {
+            // Now do any comments for the paragraph
+            XWPFCommentsDecorator comments = new XWPFCommentsDecorator(paragraph, null);
+            String commentText = comments.getCommentText();
+            if (commentText != null && commentText.length() > 0) {
+                xhtml.characters(commentText);
+            }
         }
 
         String footnameText = paragraph.getFootnoteText();

tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/XWPFEventBasedWordExtractor.java

@@ -20,10 +20,7 @@
 import javax.xml.parsers.ParserConfigurationException;
 import java.io.IOException;
 import java.io.InputStream;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 
 import org.apache.commons.io.input.CloseShieldInputStream;
 import org.apache.poi.ooxml.POIXMLDocument;
@@ -39,6 +36,7 @@
 import org.apache.poi.openxml4j.opc.PackageRelationshipCollection;
 import org.apache.poi.xwpf.usermodel.XWPFNumbering;
 import org.apache.poi.xwpf.usermodel.XWPFRelation;
+import org.apache.tika.parser.microsoft.OfficeParserConfig;
 import org.apache.tika.parser.microsoft.ooxml.OOXMLWordAndPowerPointTextHandler;
 import org.apache.tika.parser.microsoft.ooxml.ParagraphProperties;
 import org.apache.tika.parser.microsoft.ooxml.RunProperties;
@@ -60,15 +58,18 @@ public class XWPFEventBasedWordExtractor extends POIXMLTextExtractor {
     private static final Logger LOG = LoggerFactory.getLogger(XWPFEventBasedWordExtractor.class);
 
     private OPCPackage container;
+    private OfficeParserConfig config;
     private POIXMLProperties properties;
 
     public XWPFEventBasedWordExtractor(String path) throws XmlException, OpenXML4JException, IOException {
-        this(OPCPackage.open(path, PackageAccess.READ));
+        this(OPCPackage.open(path, PackageAccess.READ), new OfficeParserConfig());
     }
 
-    public XWPFEventBasedWordExtractor(OPCPackage container) throws XmlException, OpenXML4JException, IOException {
+    public XWPFEventBasedWordExtractor(OPCPackage container, OfficeParserConfig config)
+            throws XmlException, OpenXML4JException, IOException {
         super((POIXMLDocument) null);
         this.container = container;
+        this.config = config;
         this.properties = new POIXMLProperties(container);
     }
 
@@ -170,6 +171,11 @@ private void handleDocumentPart(PackagePart documentPart, StringBuilder sb) thro
                 XWPFRelation.FOOTER,
                 XWPFRelation.ENDNOTE
         }) {
+            //skip comments if we shouldn't extract them
+            if (!config.getIncludeComments() &&
+                    rel.equals(XWPFRelation.COMMENT.getRelation())) {
+                continue;
+            }
             try {
                 PackageRelationshipCollection prc = documentPart.getRelationshipsByType(rel.getRelation());
                 if (prc != null) {

tika-parsers/src/test/java/org/apache/tika/parser/microsoft/WordParserTest.java

@@ -254,7 +254,7 @@ public void testVarious() throws Exception {
         assertContains("Row 1 Col 1 Row 1 Col 2 Row 1 Col 3 Row 2 Col 1 Row 2 Col 2 Row 2 Col 3", content.replaceAll("\\s+"," "));
         assertContains("Row 1 column 1 Row 2 column 1 Row 1 column 2 Row 2 column 2", content.replaceAll("\\s+"," "));
         assertContains("This is a hyperlink", content);
-        assertContains("Here is a list:", content);
+        assertContains("Here is a list", content);
         for(int row=1;row<=3;row++) {
             //assertContains("·\tBullet " + row, content);
             //assertContains("\u00b7\tBullet " + row, content);
@@ -649,5 +649,26 @@ public void testSpecialControlCharacter() throws Exception {
         //TIKA-2459
         assertContains("Paragraph one", getXML("testWORD_specialControlCharacter1415.doc").xml);
     }
+
+    //TIKA-2900
+    @Test
+    public void testExcludeComments() throws Exception {
+        ParseContext pc = new ParseContext();
+        OfficeParserConfig officeParserConfig = new OfficeParserConfig();
+        officeParserConfig.setIncludeComments(false);
+        pc.set(OfficeParserConfig.class, officeParserConfig);
+        String xml = getXML("testWORD_comments.doc", pc).xml;
+        assertNotContained("Here is a comment", xml);
+    }
+
+    @Test
+    public void testIncludeComments() throws Exception {
+        ParseContext pc = new ParseContext();
+        OfficeParserConfig officeParserConfig = new OfficeParserConfig();
+        officeParserConfig.setIncludeComments(true);
+        pc.set(OfficeParserConfig.class, officeParserConfig);
+        String xml = getXML("testWORD_comments.doc", pc).xml;
+        assertContains("Here is a comment", xml);
+    }
 }