filter out possible corrupted characters introduced by the sentence s…

…egmenter
kermitt2 · Mar 23, 2022 · 7466fb6 · 7466fb6
1 parent 1f1ebd4
commit 7466fb6
Show file tree

Hide file tree

Showing 2 changed files with 27 additions and 3 deletions.
diff --git a/grobid-core/src/main/java/org/grobid/core/document/TEIFormatter.java b/grobid-core/src/main/java/org/grobid/core/document/TEIFormatter.java
@@ -1502,8 +1502,11 @@ public void segmentIntoSentences(Element curParagraph, List<LayoutToken> curPara
 
                 if (refPos >= pos+posInSentence && refPos <= pos+sentenceLength) {
                     Node valueNode = mapRefNodes.get(new Integer(refPos));
-                    if (pos+posInSentence < refPos)
-                        sentenceElement.appendChild(text.substring(pos+posInSentence, refPos));
+                    if (pos+posInSentence < refPos) {
+                        String local_text_chunk = text.substring(pos+posInSentence, refPos);
+                        local_text_chunk = XmlBuilderUtils.stripNonValidXMLCharacters(local_text_chunk);
+                        sentenceElement.appendChild(local_text_chunk);
+                    }
                     valueNode.detach();
                     sentenceElement.appendChild(valueNode);
                     refIndex = j;
@@ -1515,7 +1518,9 @@ public void segmentIntoSentences(Element curParagraph, List<LayoutToken> curPara
             }
 
             if (pos+posInSentence <= theSentences.get(i).end) {
-                sentenceElement.appendChild(text.substring(pos+posInSentence, theSentences.get(i).end));
+                String local_text_chunk = text.substring(pos+posInSentence, theSentences.get(i).end);
+                local_text_chunk = XmlBuilderUtils.stripNonValidXMLCharacters(local_text_chunk);
+                sentenceElement.appendChild(local_text_chunk);
                 curParagraph.appendChild(sentenceElement);
             }
         }

diff --git a/grobid-core/src/main/java/org/grobid/core/document/xml/XmlBuilderUtils.java b/grobid-core/src/main/java/org/grobid/core/document/xml/XmlBuilderUtils.java
@@ -100,4 +100,23 @@ public static void main(String[] args) throws ParsingException, IOException {
         System.out.println(toXml(e));
 
     }
+
+    public static String stripNonValidXMLCharacters(String in) {
+        StringBuffer out = new StringBuffer(); // Used to hold the output.
+        char current; // Used to reference the current character.
+
+        if (in == null || ("".equals(in))) 
+            return ""; 
+        for (int i = 0; i < in.length(); i++) {
+            current = in.charAt(i); 
+            if ((current == 0x9) ||
+                (current == 0xA) ||
+                (current == 0xD) ||
+                ((current >= 0x20) && (current <= 0xD7FF)) ||
+                ((current >= 0xE000) && (current <= 0xFFFD)) ||
+                ((current >= 0x10000) && (current <= 0x10FFFF)))
+                out.append(current);
+        }
+        return out.toString();
+    }
 }