Trying to further cut down on memory consumption. Opening issue frizb…

…og#106.
haralduna · Jul 2, 2016 · c309f22 · c309f22
1 parent e6948c2
commit c309f22
Show file tree

Hide file tree

Showing 8 changed files with 48 additions and 17 deletions.
diff --git a/src/main/java/org/gedcom4j/io/reader/AbstractEncodingSpecificReader.java b/src/main/java/org/gedcom4j/io/reader/AbstractEncodingSpecificReader.java
@@ -23,6 +23,7 @@
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.Arrays;
 import java.util.List;
 
 import org.gedcom4j.exception.GedcomParserException;
@@ -34,6 +35,14 @@
  * @author frizbog
  */
 abstract class AbstractEncodingSpecificReader {
+
+    /**
+     * A collection of entire lines (Strings to intern) when loading into StringTree's - these are the strings that
+     * appear super-frequently in files. This will help keep from making loads of duplicated copies in the heap.
+     */
+    protected final static List<String> STRINGS_TO_INTERN = Arrays
+            .asList(new String[] { "3 DATA", "1 BIRT", "1 SEX M", "1 SEX F", "1 DEAT", "1 MARR", "1 BURI", "1 EVEN", "1 RESI" });
+
     /**
      * The stream of bytes to read
      */

diff --git a/src/main/java/org/gedcom4j/io/reader/AnselReader.java b/src/main/java/org/gedcom4j/io/reader/AnselReader.java
@@ -142,7 +142,11 @@ private void addNonBlankLine() throws ParserCancelledException {
         }
         if (lineBufferIdx > 0) {
             String s = new String(lineBuffer).substring(0, lineBufferIdx);
-            result.add(s);
+            if (STRINGS_TO_INTERN.contains(s)) {
+                result.add(s.intern());
+            } else {
+                result.add(s);
+            }
         }
         linesRead++;
         if (linesRead % parser.getReadNotificationRate() == 0) {

diff --git a/src/main/java/org/gedcom4j/io/reader/AsciiReader.java b/src/main/java/org/gedcom4j/io/reader/AsciiReader.java
@@ -121,7 +121,12 @@ private void addNonBlankLine(List<String> result, StringBuilder lineBuffer) thro
             throw new ParserCancelledException("File load is cancelled");
         }
         if (lineBuffer.length() > 0) {
-            result.add(lineBuffer.toString());
+            String s = lineBuffer.toString();
+            if (STRINGS_TO_INTERN.contains(s)) {
+                result.add(s.intern());
+            } else {
+                result.add(s);
+            }
         }
         linesRead++;
         if (linesRead % parser.getReadNotificationRate() == 0) {

diff --git a/src/main/java/org/gedcom4j/io/reader/UnicodeBigEndianReader.java b/src/main/java/org/gedcom4j/io/reader/UnicodeBigEndianReader.java
@@ -129,7 +129,12 @@ private void addNonBlankLine(List<String> result, StringBuilder lineBuffer) thro
             throw new ParserCancelledException("File load is cancelled");
         }
         if (lineBuffer.length() > 0) {
-            result.add(lineBuffer.toString());
+            String s = lineBuffer.toString();
+            if (STRINGS_TO_INTERN.contains(s)) {
+                result.add(s.intern());
+            } else {
+                result.add(s);
+            }
         }
         linesRead++;
         if (linesRead % parser.getReadNotificationRate() == 0) {

diff --git a/src/main/java/org/gedcom4j/io/reader/UnicodeLittleEndianReader.java b/src/main/java/org/gedcom4j/io/reader/UnicodeLittleEndianReader.java
@@ -130,7 +130,12 @@ private void addNonBlankLine(List<String> result, StringBuilder lineBuffer) thro
             throw new ParserCancelledException("File load is cancelled");
         }
         if (lineBuffer.length() > 0) {
-            result.add(lineBuffer.toString());
+            String s = lineBuffer.toString();
+            if (STRINGS_TO_INTERN.contains(s)) {
+                result.add(s.intern());
+            } else {
+                result.add(s);
+            }
         }
         linesRead++;
         if (linesRead % parser.getReadNotificationRate() == 0) {

diff --git a/src/main/java/org/gedcom4j/io/reader/Utf8Reader.java b/src/main/java/org/gedcom4j/io/reader/Utf8Reader.java
@@ -88,7 +88,11 @@ protected List<String> load() throws IOException, ParserCancelledException {
                     throw new ParserCancelledException("File load is cancelled");
                 }
                 if (s.length() != 0) {
-                    result.add(s);
+                    if (STRINGS_TO_INTERN.contains(s)) {
+                        result.add(s.intern());
+                    } else {
+                        result.add(s);
+                    }
                 }
                 linesRead++;
                 if (linesRead % parser.getReadNotificationRate() == 0) {

diff --git a/src/main/java/org/gedcom4j/parser/LinePieces.java b/src/main/java/org/gedcom4j/parser/LinePieces.java
@@ -55,11 +55,6 @@ class LinePieces {
      */
     private int currCharIdx;
 
-    /**
-     * The line
-     */
-    private final String line;
-
     /** The characters in the line */
     private final char[] chars;
 
@@ -75,9 +70,7 @@ class LinePieces {
      */
     LinePieces(String lineToParse, int lineNum) throws GedcomParserException {
 
-        line = lineToParse;
-
-        chars = line.toCharArray();
+        chars = lineToParse.toCharArray();
 
         processLevel(lineNum);
 
@@ -98,7 +91,7 @@ class LinePieces {
      */
     private void processLevel(int lineNum) throws GedcomParserException {
         try {
-            char c2 = line.charAt(1); // 2nd character in line
+            char c2 = chars[1]; // 2nd character in line
 
             currCharIdx = -1;
 
@@ -112,9 +105,11 @@ private void processLevel(int lineNum) throws GedcomParserException {
                 currCharIdx = 3; // Continue parsing at 4th character in line
             }
         } catch (NumberFormatException e) {
-            throw new GedcomParserException("Line " + lineNum + " does not begin with a 1 or 2 digit number for the level followed by a space: " + line, e);
+            throw new GedcomParserException(
+                    "Line " + lineNum + " does not begin with a 1 or 2 digit number for the level followed by a space: " + new String(chars), e);
         } catch (IndexOutOfBoundsException e) {
-            throw new GedcomParserException("Line " + lineNum + " does not begin with a 1 or 2 digit number for the level followed by a space: " + line, e);
+            throw new GedcomParserException(
+                    "Line " + lineNum + " does not begin with a 1 or 2 digit number for the level followed by a space: " + new String(chars), e);
         }
     }
 
@@ -123,7 +118,7 @@ private void processLevel(int lineNum) throws GedcomParserException {
      */
     private void processRemainder() {
         if (currCharIdx < chars.length) {
-            remainder = line.substring(currCharIdx + 1);
+            remainder = new String(chars, currCharIdx + 1, chars.length - currCharIdx - 1);
         }
     }
 

diff --git a/src/main/java/org/gedcom4j/parser/StringTreeBuilder.java b/src/main/java/org/gedcom4j/parser/StringTreeBuilder.java
@@ -31,6 +31,9 @@ static String leftTrim(String line) {
         if (line == "") {
             return "";
         }
+        if (!Character.isWhitespace(line.charAt(0))) {
+            return line;
+        }
         for (int i = 0; i < line.length(); i++) {
             if (!Character.isWhitespace(line.charAt(i))) {
                 return line.substring(i);
@@ -148,6 +151,7 @@ private void addNewNode(int lineNum, String line) throws GedcomParserException {
         treeForCurrentLine.id = lp.id;
         treeForCurrentLine.tag = lp.tag.intern();
         treeForCurrentLine.value = lp.remainder;
+        lp = null;
 
         StringTree addTo = null;
         if (treeForCurrentLine.level == 0) {