Skip to content

Commit 2d199aa

Browse files
tballisontballison
authored
TIKA-4430 -- improve extraction of metadata from xls (#2240)
1 parent 27e8c30 commit 2d199aa

3 files changed

Lines changed: 94 additions & 2 deletions

File tree

‎tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ExcelExtractor.java‎

Lines changed: 79 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,10 @@
2424
import java.util.List;
2525
import java.util.Locale;
2626
import java.util.Map;
27+
import java.util.Set;
2728
import java.util.SortedMap;
2829
import java.util.TreeMap;
30+
import java.util.TreeSet;
2931

3032
import org.apache.poi.ddf.EscherBSERecord;
3133
import org.apache.poi.ddf.EscherBlipRecord;
@@ -39,6 +41,7 @@
3941
import org.apache.poi.hssf.record.BOFRecord;
4042
import org.apache.poi.hssf.record.BoundSheetRecord;
4143
import org.apache.poi.hssf.record.CellValueRecordInterface;
44+
import org.apache.poi.hssf.record.ColumnInfoRecord;
4245
import org.apache.poi.hssf.record.CountryRecord;
4346
import org.apache.poi.hssf.record.DateWindow1904Record;
4447
import org.apache.poi.hssf.record.DrawingGroupRecord;
@@ -51,9 +54,12 @@
5154
import org.apache.poi.hssf.record.HyperlinkRecord;
5255
import org.apache.poi.hssf.record.LabelRecord;
5356
import org.apache.poi.hssf.record.LabelSSTRecord;
57+
import org.apache.poi.hssf.record.NoteRecord;
5458
import org.apache.poi.hssf.record.NumberRecord;
59+
import org.apache.poi.hssf.record.ProtectRecord;
5560
import org.apache.poi.hssf.record.RKRecord;
5661
import org.apache.poi.hssf.record.Record;
62+
import org.apache.poi.hssf.record.RowRecord;
5763
import org.apache.poi.hssf.record.SSTRecord;
5864
import org.apache.poi.hssf.record.StringRecord;
5965
import org.apache.poi.hssf.record.TextObjectRecord;
@@ -73,8 +79,10 @@
7379
import org.apache.tika.exception.TikaException;
7480
import org.apache.tika.io.TikaInputStream;
7581
import org.apache.tika.metadata.Metadata;
82+
import org.apache.tika.metadata.Office;
7683
import org.apache.tika.parser.ParseContext;
7784
import org.apache.tika.sax.XHTMLContentHandler;
85+
import org.apache.tika.utils.StringUtils;
7886

7987
/**
8088
* Excel parser implementation which uses POI's Event API
@@ -188,6 +196,7 @@ protected void parse(DirectoryNode root, XHTMLContentHandler xhtml, Locale local
188196
new TikaHSSFListener(workbookEntryName, xhtml, locale, this, officeParserConfig);
189197
listener.processFile(root, isListenForAllRecords());
190198
listener.throwStoredException();
199+
updateMetadata(listener);
191200

192201
for (Entry entry : root) {
193202
if (entry.getName().startsWith("MBD") && entry instanceof DirectoryEntry) {
@@ -200,6 +209,36 @@ protected void parse(DirectoryNode root, XHTMLContentHandler xhtml, Locale local
200209
}
201210
}
202211

212+
private void updateMetadata(TikaHSSFListener listener) {
213+
if (listener.hasProtectedSheet) {
214+
parentMetadata.set(Office.PROTECTED_WORKSHEET, true);
215+
}
216+
if (listener.hasHiddenColumn) {
217+
parentMetadata.set(Office.HAS_HIDDEN_COLUMNS, true);
218+
}
219+
if (listener.hasHiddenRow) {
220+
parentMetadata.set(Office.HAS_HIDDEN_ROWS, true);
221+
}
222+
if (! listener.commentAuthors.isEmpty()) {
223+
for (String author : listener.commentAuthors) {
224+
parentMetadata.add(Office.COMMENT_PERSONS, author);
225+
}
226+
parentMetadata.set(Office.HAS_COMMENTS, true);
227+
}
228+
if (! listener.hiddenSheets.isEmpty()) {
229+
for (String sheetName : listener.hiddenSheets) {
230+
parentMetadata.add(Office.HIDDEN_SHEET_NAMES, sheetName);
231+
}
232+
parentMetadata.set(Office.HAS_HIDDEN_SHEETS, true);
233+
}
234+
if (! listener.veryHiddenSheets.isEmpty()) {
235+
for (String sheetName : listener.veryHiddenSheets) {
236+
parentMetadata.add(Office.VERY_HIDDEN_SHEET_NAMES, sheetName);
237+
}
238+
parentMetadata.set(Office.HAS_VERY_HIDDEN_SHEETS, true);
239+
}
240+
}
241+
203242
// ======================================================================
204243

205244
/**
@@ -266,7 +305,14 @@ private static class TikaHSSFListener implements HSSFListener {
266305
* depend on continue records that aren't always
267306
* contiguous. Collect them for later processing.
268307
*/
269-
private List<DrawingGroupRecord> drawingGroups = new ArrayList<>();
308+
private final List<DrawingGroupRecord> drawingGroups = new ArrayList<>();
309+
310+
private final List<String> hiddenSheets = new ArrayList<>();
311+
private final List<String> veryHiddenSheets = new ArrayList<>();
312+
private final Set<String> commentAuthors = new TreeSet<>();
313+
private boolean hasHiddenColumn = false;
314+
private boolean hasHiddenRow = false;
315+
private boolean hasProtectedSheet = false;
270316

271317
/**
272318
* Construct a new listener instance outputting parsed data to
@@ -328,6 +374,10 @@ public void processFile(DirectoryNode root, boolean listenForAllRecords)
328374
hssfRequest.addListener(formatListener, FormatRecord.sid);
329375
hssfRequest.addListener(formatListener, ExtendedFormatRecord.sid);
330376
hssfRequest.addListener(formatListener, DrawingGroupRecord.sid);
377+
hssfRequest.addListener(formatListener, ProtectRecord.sid);
378+
hssfRequest.addListener(formatListener, ColumnInfoRecord.sid);
379+
hssfRequest.addListener(formatListener, RowRecord.sid);
380+
hssfRequest.addListener(formatListener, NoteRecord.sid);
331381
if (extractor.officeParserConfig.isIncludeHeadersAndFooters()) {
332382
hssfRequest.addListener(formatListener, HeaderRecord.sid);
333383
hssfRequest.addListener(formatListener, FooterRecord.sid);
@@ -419,6 +469,12 @@ private void internalProcessRecord(Record record)
419469

420470
case BoundSheetRecord.sid: // Worksheet index record
421471
BoundSheetRecord boundSheetRecord = (BoundSheetRecord) record;
472+
if (boundSheetRecord.isHidden()) {
473+
hiddenSheets.add(boundSheetRecord.getSheetname());
474+
}
475+
if (boundSheetRecord.isVeryHidden()) {
476+
veryHiddenSheets.add(boundSheetRecord.getSheetname());
477+
}
422478
sheetNames.add(boundSheetRecord.getSheetname());
423479
break;
424480

@@ -524,6 +580,28 @@ private void internalProcessRecord(Record record)
524580
addTextCell(record, footerRecord.getText());
525581
}
526582
break;
583+
case ProtectRecord.sid:
584+
if (((ProtectRecord)record).getProtect()) {
585+
//TODO -- associate this worksheet name
586+
hasProtectedSheet = true;
587+
}
588+
break;
589+
case ColumnInfoRecord.sid:
590+
if (((ColumnInfoRecord)record).getHidden()) {
591+
hasHiddenColumn = true;
592+
}
593+
break;
594+
case NoteRecord.sid:
595+
String author = ((NoteRecord)record).getAuthor();
596+
if (!StringUtils.isBlank(author)) {
597+
commentAuthors.add(author);
598+
}
599+
break;
600+
case RowRecord.sid:
601+
if (((RowRecord)record).getZeroHeight()) {
602+
hasHiddenRow = true;
603+
}
604+
break;
527605
}
528606

529607
previousSid = record.getSid();
@@ -680,7 +758,6 @@ public TikaFormatTrackingHSSFListener(HSSFListener childListener, Locale locale)
680758

681759
@Override
682760
public void processRecord(Record record) {
683-
// System.out.println(record.getClass() + " : "+record.toString());
684761
super.processRecord(record);
685762
}
686763

‎tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/ExcelParserTest.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323

2424
import java.io.InputStream;
2525
import java.text.DecimalFormatSymbols;
26+
import java.util.List;
2627
import java.util.Locale;
2728

2829
import org.apache.poi.util.LocaleUtil;
@@ -577,4 +578,18 @@ public void testDateFormat() throws Exception {
577578
assertContains("1996-08-10", xml);
578579
}
579580
}
581+
582+
@Test
583+
public void testExtraMetadata() throws Exception {
584+
List<Metadata> metadataList = getRecursiveMetadata("testEXCEL_extra_metadata.xls");
585+
Metadata m = metadataList.get(0);
586+
assertEquals("Unknown Author", m.getValues(Office.COMMENT_PERSONS)[0]);
587+
assertEquals("true", m.get(Office.HAS_HIDDEN_COLUMNS));
588+
assertEquals("true", m.get(Office.HAS_HIDDEN_ROWS));
589+
assertEquals("true", m.get(Office.PROTECTED_WORKSHEET));
590+
assertEquals("hidden-sheet", m.getValues(Office.HIDDEN_SHEET_NAMES)[0]);
591+
assertEquals("very-hidden-sheet", m.getValues(Office.VERY_HIDDEN_SHEET_NAMES)[0]);
592+
assertEquals("true", m.get(Office.HAS_COMMENTS));
593+
assertEquals("true", m.get(Office.HAS_HIDDEN_COLUMNS));
594+
}
580595
}

‎tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testEXCEL_extra_metadata.xls‎

12.5 KB
Binary file not shown.

0 commit comments

Comments
 (0)