Filter out empty/whitespace-only regions (fixes #105)

dbmdz · May 11, 2020 · 408a15c · 408a15c
1 parent ca3456d
commit 408a15c
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 0 deletions.
diff --git a/docs/changes.md b/docs/changes.md
@@ -33,6 +33,7 @@ This is a major release with a focus on compatibility and performance.
 - Log warnings during source pointer parsing
 - Filter out empty files during indexing
 - Add new documentation section on performance tuning
+- Empty regions or regions with only whitespace are no longer included in the output
 
 ## 0.3.1 (2019-07-26)
 

diff --git a/src/main/java/de/digitalcollections/solrocr/formats/OcrPassageFormatter.java b/src/main/java/de/digitalcollections/solrocr/formats/OcrPassageFormatter.java
@@ -198,6 +198,7 @@ protected OcrSnippet parseFragment(String ocrFragment, OcrPage page) {
     String highlightedText = getTextFromXml(ocrFragment);
     List<OcrBox> snippetRegions = byColumns.stream()
         .map(this::determineSnippetRegion)
+        .filter(r -> !r.getText().isEmpty() && !r.getText().trim().isEmpty())
         .collect(Collectors.toList());
     Set<String> snippetPageIds = snippetRegions.stream()
         .map(OcrBox::getPageId).collect(Collectors.toSet());