more robustness wrt sentence segmenter output

kermitt2 · Mar 26, 2022 · 79ebdd7 · 79ebdd7
1 parent d8e2afd
commit 79ebdd7
Showing 1 changed file with 5 additions and 2 deletions.
diff --git a/grobid-core/src/main/java/org/grobid/core/document/TEIFormatter.java b/grobid-core/src/main/java/org/grobid/core/document/TEIFormatter.java
@@ -1502,8 +1502,11 @@ public void segmentIntoSentences(Element curParagraph, List<LayoutToken> curPara
 
                 if (refPos >= pos+posInSentence && refPos <= pos+sentenceLength) {
                     Node valueNode = mapRefNodes.get(new Integer(refPos));
-                    if (pos+posInSentence < refPos)
-                        sentenceElement.appendChild(text.substring(pos+posInSentence, refPos));
+                    if (pos+posInSentence < refPos) {
+                        String local_text_chunk = text.substring(pos+posInSentence, refPos);
+                        local_text_chunk = XmlBuilderUtils.stripNonValidXMLCharacters(local_text_chunk);
+                        sentenceElement.appendChild(local_text_chunk);
+                    }
                     valueNode.detach();
                     sentenceElement.appendChild(valueNode);
                     refIndex = j;