Permalink
Browse files

Improved parsing support for OOXML spreadsheets (.xlsx)

As reported edycop in mantis 765 (
http://mantis.tokeek.de/view.php?id=765 ), parsing of xlsx files was
quite incomplete.
Now properly support "Shared String Table" entry in Office Open XML
spreadsheets, an also detect embedded URLs.

Integrating the Apache poi-ooxml library could be an option for finer
OOXML formats support, but their SAX style parsing example (
http://poi.apache.org/spreadsheet/how-to.html#xssf_sax_api ) tends to
show that a custom SAX handler is still efficient for lightweight and
low memory footprint processing.
  • Loading branch information...
luccioman committed Aug 21, 2017
1 parent 51a4e03 commit e41d046a9d4cdbcfbbd297da817a853b8415f058
@@ -44,6 +44,7 @@
import javax.xml.parsers.SAXParserFactory;
import net.yacy.cora.document.encoding.UTF8;
import net.yacy.cora.document.id.AnchorURL;
import net.yacy.cora.document.id.DigestURL;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.document.AbstractParser;
@@ -52,6 +53,8 @@
import net.yacy.document.VocabularyScraper;
import net.yacy.document.parser.xml.ODContentHandler;
import net.yacy.document.parser.xml.ODMetaHandler;
import net.yacy.document.parser.xml.OOXMLSharedStringsHandler;
import net.yacy.document.parser.xml.OOXMLSpreeadsheetHandler;
import net.yacy.kelondro.io.CharBuffer;
import net.yacy.kelondro.util.FileUtils;
@@ -108,21 +111,53 @@ private static SAXParser getParser() throws SAXException {
// opening the file as zip file
final ZipFile zipFile= new ZipFile(dest);
final Enumeration<? extends ZipEntry> zipEnum = zipFile.entries();
final Set<AnchorURL> detectedURLs = new HashSet<>();
/* Handle first any eventual spreadsheet shared strings table
* As stated by the standard : "A package shall contain exactly one Shared String Table part," */
final ZipEntry sharedStringsEntry = zipFile.getEntry(OOXMLSharedStringsHandler.ENTRY_NAME);
final List<String> sharedStrings = new ArrayList<>();
if (sharedStringsEntry != null) {
// extract data
try (final InputStream zipFileEntryStream = zipFile.getInputStream(sharedStringsEntry)) {
final SAXParser saxParser = getParser();
saxParser.parse(zipFileEntryStream, new OOXMLSharedStringsHandler(sharedStrings));
}
}
// looping through all containing files
while (zipEnum.hasMoreElements()) {
// get next zip file entry
final ZipEntry zipEntry= zipEnum.nextElement();
final String entryName = zipEntry.getName();
if(entryName.startsWith("xl/worksheets/sheet")) {
if(writer == null) {
// create a writer for output
writer = new CharBuffer(odtParser.MAX_DOCSIZE, (int) zipEntry.getSize());
}
// extract data
final InputStream zipFileEntryStream = zipFile.getInputStream(zipEntry);
try {
final SAXParser saxParser = getParser();
saxParser.parse(zipFileEntryStream, new OOXMLSpreeadsheetHandler(sharedStrings, writer, detectedURLs));
// content.xml contains the document content in xml format
if (entryName.equals("word/document.xml")
|| entryName.startsWith("ppt/slides/slide")
|| entryName.startsWith("xl/worksheets/sheet")) {
// close readers and writers
} finally {
zipFileEntryStream.close();
}
} else if (entryName.equals("word/document.xml")
|| entryName.startsWith("ppt/slides/slide")) {
// create a writer for output
writer = new CharBuffer(odtParser.MAX_DOCSIZE, (int) zipEntry.getSize());
if(writer == null) {
// create a writer for output
writer = new CharBuffer(odtParser.MAX_DOCSIZE, (int) zipEntry.getSize());
}
// extract data
final InputStream zipFileEntryStream = zipFile.getInputStream(zipEntry);
@@ -186,7 +221,7 @@ private static SAXParser getParser() throws SAXException {
descriptions,
0.0d, 0.0d,
contentBytes,
null,
detectedURLs,
null,
null,
false,
@@ -35,7 +35,6 @@
import org.xml.sax.helpers.DefaultHandler;
import net.yacy.cora.document.id.AnchorURL;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.document.parser.html.ContentScraper;
/**
@@ -150,7 +149,7 @@ public void startElement(String uri, String localName, String qName, Attributes
public void characters(final char ch[], final int start, final int length) throws SAXException {
try {
if(this.currentElementTextChunks == 0 && this.documentHasText) {
/* We are but on the first text chunk of the element (not on the first text chunk of the whole document),
/* We are on the first text chunk of the element (not on the first text chunk of the whole document),
* or on the first text chunk after processing nested elements :
* if necessary we add a space to separate text content of different elements */
if(length > 0 && !this.lastAppendedIsSpace && !Character.isWhitespace(ch[0])) {
@@ -167,14 +166,14 @@ public void characters(final char ch[], final int start, final int length) throw
this.documentHasText = true;
this.lastAppendedIsSpace = Character.isWhitespace(ch[length - 1]);
}
} catch (final IOException ignored) {
ConcurrentLog.logException(ignored);
} catch (final IOException ioe) {
throw new SAXException("Error while appending characters to the output writer", ioe);
}
}
/**
* When the eventual element text doesn't end with a terminal punctuation character,
* add a period ('.' character) to help future SentenceReader work.
* Perform URLs detection on the ending element text
* @throws SAXException when whe maxURLs limit has been reached
*/
@Override
public void endElement(String uri, String localName, String qName) throws SAXException {
@@ -0,0 +1,120 @@
// OOXMLSharedStringsHandler.java
// ---------------------------
// Copyright 2017 by luccioman; https://github.com/luccioman
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package net.yacy.document.parser.xml;
import java.io.IOException;
import java.util.List;
import org.apache.commons.io.input.ClosedInputStream;
import org.xml.sax.Attributes;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.DefaultHandler;
/**
* SAX handler for Office Open XML SpreadSheet xl/sharedStrings.xml files.
*
* @author luccioman
* @see <a href=
* "http://www.ecma-international.org/publications/standards/Ecma-376.htm">Ecma
* Standard for Office Open XML File Formats</a>
*
*/
public class OOXMLSharedStringsHandler extends DefaultHandler {
/** The entry name of a shared strings table in an OOXML container */
public static final String ENTRY_NAME = "xl/sharedStrings.xml";
/** Name of a shared string tag */
private static final String SHARED_STRING_TAG = "t";
/** Shared strings list */
private final List<String> sharedStrings;
/** Currently parsed string builder. */
private StringBuilder currentString;
/**
* @param sharedStrings
* the mutable list of shared strings to fill
* @throws IllegalArgumentException
* when a parameter is null
*/
public OOXMLSharedStringsHandler(final List<String> sharedStrings) throws IllegalArgumentException {
if (sharedStrings == null) {
throw new IllegalArgumentException("sharedStrings list must not be null");
}
this.sharedStrings = sharedStrings;
}
/**
* @return an empty source to prevent the SAX parser opening an unwanted
* connection to resolve an external entity
*/
@Override
public InputSource resolveEntity(String publicId, String systemId) throws IOException, SAXException {
return new InputSource(new ClosedInputStream());
}
@Override
public void startDocument() throws SAXException {
this.currentString = new StringBuilder();
}
@Override
public void startElement(final String uri, final String localName, final String qName, final Attributes attributes)
throws SAXException {
if (SHARED_STRING_TAG.equals(qName)) {
this.currentString.setLength(0);
}
}
/**
* Append characters to the current string builder. May be called multiple times
* before obtaining the whole current element string.
*/
@Override
public void characters(final char ch[], final int start, final int length) throws SAXException {
this.currentString.append(ch, start, length);
}
/**
* Add the current string content to the list when ending a shared string
* element
*/
@Override
public void endElement(String uri, String localName, String qName) throws SAXException {
if (SHARED_STRING_TAG.equals(qName)) {
final String sharedString = this.currentString.toString();
this.sharedStrings.add(sharedString);
this.currentString.setLength(0);
}
}
@Override
public void endDocument() throws SAXException {
/* Release the StringBuilder now useless */
this.currentString = null;
}
}
Oops, something went wrong.

0 comments on commit e41d046

Please sign in to comment.