Merge pull request #1142 from stanfordnlp/sr_custom_serialization

Sr custom serialization

Author: AngledLuffa

scripts/srparser/Makefile

@@ -144,7 +144,7 @@ englishSR.ser.gz:
 englishSR.beam.ser.gz:
 	@echo Training $@
 	@echo Will test on $(ENGLISH_TEST)
-	java -mx50g edu.stanford.nlp.parser.shiftreduce.ShiftReduceParser -trainTreebank $(ENGLISH_TRAIN) -devTreebank $(ENGLISH_DEV) -serializedPath $@ $(DEFAULT_OPTIONS) -preTag -taggerSerializedFile $(ENGLISH_TAGGER) -tlpp $(ENGLISH_TLPP) $(TRAIN_BEAM) $(AUGMENT_LESS) > $@.out 2>&1
+	java -mx80g edu.stanford.nlp.parser.shiftreduce.ShiftReduceParser -trainTreebank $(ENGLISH_TRAIN) -devTreebank $(ENGLISH_DEV) -serializedPath $@ $(DEFAULT_OPTIONS) -preTag -taggerSerializedFile $(ENGLISH_TAGGER) -tlpp $(ENGLISH_TLPP) $(TRAIN_BEAM) $(AUGMENT_LESS) > $@.out 2>&1
 	java -mx5g edu.stanford.nlp.parser.shiftreduce.ShiftReduceParser $(TEST_ARGS) -testTreebank $(ENGLISH_TEST) -serializedPath $@ -preTag -taggerSerializedFile $(ENGLISH_TAGGER) >> $@.out 2>&1
 
 frenchSR.ser.gz:

src/edu/stanford/nlp/io/ByteArrayUtils.java

@@ -0,0 +1,60 @@
+package edu.stanford.nlp.io;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+
+/**
+ * Static methods for putting shorts, ints, and longs into a ByteArrayOutputStream using bit fiddling
+ *
+ * @author John Bauer
+ */
+public class ByteArrayUtils {
+  static public short readShort(ByteArrayInputStream bin) {
+    int high = ((bin.read() & 0x000000FF) << 8);
+    int low = (bin.read() & 0x000000FF);
+    return (short) ((high | low) & 0x0000FFFF);
+  }
+
+  static public void writeShort(ByteArrayOutputStream bout, short val) {
+    bout.write((byte)((val >> 8) & 0xff));
+    bout.write((byte)(val & 0xff));
+  }
+
+  static public int readInt(ByteArrayInputStream bin) {
+    int b24 = ((bin.read() & 0x000000FF) << 24);
+    int b16 = ((bin.read() & 0x000000FF) << 16);
+    int b8 = ((bin.read() & 0x000000FF) << 8);
+    int b0 = (bin.read() & 0x000000FF);
+    return b24 | b16 | b8 | b0;
+  }
+
+  static public void writeInt(ByteArrayOutputStream bout, int val) {
+    bout.write((byte)((val >> 24) & 0xff));
+    bout.write((byte)((val >> 16) & 0xff));
+    bout.write((byte)((val >> 8) & 0xff));
+    bout.write((byte)(val & 0xff));
+  }
+
+  static public long readLong(ByteArrayInputStream bin) {
+    long b56 = ((long) (bin.read() & 0x000000FF)) << 56;
+    long b48 = ((long) (bin.read() & 0x000000FF)) << 48;
+    long b40 = ((long) (bin.read() & 0x000000FF)) << 40;
+    long b32 = ((long) (bin.read() & 0x000000FF)) << 32;
+    long b24 = ((long) (bin.read() & 0x000000FF)) << 24;
+    long b16 = ((long) (bin.read() & 0x000000FF)) << 16;
+    long b8 = ((long) (bin.read() & 0x000000FF)) << 8;
+    long b0 = ((long) (bin.read() & 0x000000FF));
+    return b56 | b48 | b40 | b32 | b24 | b16 | b8 | b0;
+  }
+
+  static public void writeLong(ByteArrayOutputStream bout, long val) {
+    bout.write((byte)((val >> 56) & 0xff));
+    bout.write((byte)((val >> 48) & 0xff));
+    bout.write((byte)((val >> 40) & 0xff));
+    bout.write((byte)((val >> 32) & 0xff));
+    bout.write((byte)((val >> 24) & 0xff));
+    bout.write((byte)((val >> 16) & 0xff));
+    bout.write((byte)((val >> 8) & 0xff));
+    bout.write((byte)(val & 0xff));
+  }
+}

src/edu/stanford/nlp/parser/shiftreduce/BinaryTransition.java

@@ -232,12 +232,11 @@ public boolean equals(Object o) {
 
   @Override
   public int hashCode() {
-    // TODO: fix the hashcode for the side?  would require rebuilding all models
     switch(side) {
     case LEFT:
       return 97197711 ^ label.hashCode();
     case RIGHT:
-      return 97197711 ^ label.hashCode();
+      return 85635467 ^ label.hashCode();
     default:
       throw new IllegalArgumentException("Unknown side " + side);
     }

src/edu/stanford/nlp/parser/shiftreduce/PerceptronModel.java

@@ -44,13 +44,13 @@ public class PerceptronModel extends BaseModel  {
 
   private float learningRate = 1.0f;
 
-  Map<String, Weight> featureWeights;
+  WeightMap featureWeights;
   final FeatureFactory featureFactory;
 
   public PerceptronModel(ShiftReduceOptions op, Index<Transition> transitionIndex,
                          Set<String> knownStates, Set<String> rootStates, Set<String> rootOnlyStates) {
     super(op, transitionIndex, knownStates, rootStates, rootOnlyStates);
-    this.featureWeights = Generics.newHashMap();
+    this.featureWeights = new WeightMap();
 
     String[] classes = op.featureFactoryClass.split(";");
     if (classes.length == 1) {
@@ -74,7 +74,7 @@ public PerceptronModel(PerceptronModel other) {
     super(other);
     this.featureFactory = other.featureFactory;
 
-    this.featureWeights = Generics.newHashMap();
+    this.featureWeights = new WeightMap();
     for (String feature : other.featureWeights.keySet()) {
       featureWeights.put(feature, new Weight(other.featureWeights.get(feature)));
     }
@@ -110,7 +110,7 @@ public void averageModels(Collection<PerceptronModel> models) {
       }
     }
 
-    featureWeights = Generics.newHashMap();
+    featureWeights = new WeightMap();
     for (String feature : features) {
       featureWeights.put(feature, new Weight());
     }

src/edu/stanford/nlp/parser/shiftreduce/Weight.java

@@ -1,9 +1,13 @@
 package edu.stanford.nlp.parser.shiftreduce;
 
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
 import java.io.Serializable;
 
+import edu.stanford.nlp.io.ByteArrayUtils;
 import edu.stanford.nlp.util.ArrayUtils;
 
+
 /**
  * Stores one row of the sparse matrix which makes up the multiclass perceptron.
  *
@@ -23,63 +27,74 @@
  */
 
 public class Weight implements Serializable {
+  static final short[] EMPTY = {};
+
   public Weight() {
-    packed = null;
+    packed = EMPTY;
   }
 
   public Weight(Weight other) {
     if (other.size() == 0) {
-      packed = null;
+      packed = EMPTY;
       return;
     }
     packed = ArrayUtils.copy(other.packed);
     condense();
   }
 
   public int size() {
-    if (packed == null) {
-      return 0;
-    }
-    return packed.length;
+    // TODO: find a fast way of doing this... we know it's a multiple of 3 after all
+    return packed.length / 3;
   }
 
-  private int unpackIndex(int i) {
-    long pack = packed[i];
-    return (int) (pack >>> 32);
+  private short unpackIndex(int i) {
+    return packed[i * 3];
   }
 
   private float unpackScore(int i) {
-    long pack = packed[i];
-    return Float.intBitsToFloat((int) (pack & 0xFFFFFFFF));
+    i = i * 3 + 1;
+    final int high = ((int) packed[i++]) << 16;
+    final int low = packed[i] & 0x0000FFFF;
+    return Float.intBitsToFloat(high | low);
   }
 
-  private static long packedValue(int index, float score) {
-    long pack = ((long) (Float.floatToIntBits(score))) & 0x00000000FFFFFFFFL;
-    pack = pack | (((long) index) << 32);
-    return pack;
-  }
-
-  private static void pack(long[] packed, int i, int index, float score) {
-    packed[i] = packedValue(index, score);
+  private static void pack(short[] packed, int i, int index, float score) {
+    if (i > Short.MAX_VALUE) {
+      throw new ArithmeticException("How did you make an index with 30,000 weights??");
+    }
+    int pos = i * 3;
+    packed[pos++] = (short) index;
+    final int bits = Float.floatToIntBits(score);
+    packed[pos++] = (short) ((bits & 0xFFFF0000) >> 16);
+    packed[pos] = (short) (bits & 0x0000FFFF);
   }
 
   private void pack(int i, int index, float score) {
-    packed[i] = packedValue(index, score);
+    if (i > Short.MAX_VALUE) {
+      throw new ArithmeticException("How did you make an index with 30,000 weights??");
+    }
+    int pos = i * 3;
+    packed[pos++] = (short) index;
+    final int bits = Float.floatToIntBits(score);
+    packed[pos++] = (short) ((bits & 0xFFFF0000) >> 16);
+    packed[pos] = (short) (bits & 0x0000FFFF);
   }
 
   public void score(float[] scores) {
-    final int length = size();
-    if (length > scores.length) {
+    if (packed.length > scores.length * 3) {
       throw new AssertionError("Called with an array of scores too small to fit");
     }
-    for (int i = 0; i < length; ++i) {
+    for (int i = 0; i < packed.length; ) {
       // Since this is the critical method, we optimize it even further.
       // We could do this:
       // int index = unpackIndex; float score = unpackScore;
-      // That results in an extra array lookup
-      final long pack = packed[i];
-      final int index = (int) (pack >>> 32);
-      final float score = Float.intBitsToFloat((int) (pack & 0xFFFFFFFF));
+      // That results in extra operations
+      final short index = packed[i++];
+      final int high = ((int) packed[i++]) << 16;
+      final int low = packed[i++] & 0x0000FFFF;
+      final int bits = high | low;
+      // final int bits = (((int) packed[i++]) << 16) | (packed[i++] & 0x0000FFFF);
+      final float score = Float.intBitsToFloat(bits);
       scores[index] += score;
     }
   }
@@ -98,7 +113,7 @@ public void addScaled(Weight other, float scale) {
   void condense() {
     // threshold is in case floating point math makes a feature we
     // don't care about exist
-    if (packed == null) {
+    if (packed == null || packed.length == 0) {
       return;
     }
 
@@ -111,15 +126,15 @@ void condense() {
     }
 
     if (nonzero == 0) {
-      packed = null;
+      packed = EMPTY;
       return;
     }
 
     if (nonzero == length) {
       return;
     }
 
-    long[] newPacked = new long[nonzero];
+    short[] newPacked = new short[nonzero * 3];
     int j = 0;
     for (int i = 0; i < length; ++i) {
       if (Math.abs(unpackScore(i)) <= THRESHOLD) {
@@ -152,23 +167,23 @@ public void updateWeight(int index, float increment) {
       return;
     }
 
-    if (packed == null) {
-      packed = new long[1];
+    if (packed == null || packed.length == 0) {
+      packed = new short[3];
       pack(0, index, increment);
       return;
     }
 
     final int length = size();
     for (int i = 0; i < length; ++i) {
       if (unpackIndex(i) == index) {
-        float score = unpackScore(i);
+        final float score = unpackScore(i);
         pack(i, index, score + increment);
         return;
       }
     }
 
-    long[] newPacked = new long[length + 1];
-    for (int i = 0; i < length; ++i) {
+    short[] newPacked = new short[packed.length + 3];
+    for (int i = 0; i < packed.length; ++i) {
       newPacked[i] = packed[i];
     }
     pack(newPacked, length, index, increment);
@@ -231,15 +246,33 @@ void l2Reg(float reg) {
   public String toString() {
     StringBuilder builder = new StringBuilder();
     final int length = size();
+    builder.append("Weight(");
     for (int i = 0; i < length; ++i) {
-      if (i > 0) builder.append("   ");
+      if (i > 0) builder.append("  ");
       builder.append(unpackIndex(i) + "=" + unpackScore(i));
     }
+    builder.append(")");
     return builder.toString();
   }
 
-  private long[] packed;
+  private short[] packed;
 
-  private static final long serialVersionUID = 1;
+  void writeBytes(ByteArrayOutputStream bout) {
+    ByteArrayUtils.writeInt(bout, packed.length);
+    for (int i = 0; i < packed.length; ++i) {
+      ByteArrayUtils.writeShort(bout, packed[i]);
+    }
+  }
+
+  static Weight readBytes(ByteArrayInputStream bin) {
+    int len = ByteArrayUtils.readInt(bin);
+    Weight weight = new Weight();
+    weight.packed = new short[len];
+    for (int i = 0; i < len; ++i) {
+      weight.packed[i] = ByteArrayUtils.readShort(bin);
+    }
+    return weight;
+  }
 
+  private static final long serialVersionUID = 3;
 }