Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP

Loading…

Optimized serialization for JDBM3 #72

Merged
merged 2 commits into from

2 participants

@romix

As I've written in the issue related to the serialization of enums, I did some improvements related to serialization:
1) Enums can be serialized now
2) Arrays serialization is improved, as it was failing for some of my private tests
3) Speed of serialization/deserialization is about 80-100 times faster than before. This is achieved by caching results of reflection-calls as much as possible and reusing them. Also the number of lookups by class names and field names was reduced.

With these changes JDBM3 still passes all its JUnit tests. With the added speed improvements, JDBM3 is now producing on my tests the most compact non-compressed and second-smallest compressed serialized representation, when compared to Java serialization, Kryo and protostuff. More over, it is now the fastest (!) serialization out of all mentioned frameworks.

P.S. The serialization part of JDBM3 is so good now that it would make sense to create a small serialization library out of it, as it is quite comparable to Kryo and protostuff now.

romix added some commits
@jankotek jankotek merged commit 6b4b390 into jankotek:master
@jankotek jankotek referenced this pull request from a commit
@jankotek Fixed issue #72
DBMaker.closeOnExit option causes memory 'leak'
a450a10
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Commits on Apr 24, 2012
  1. Add a header for IdentityHashMaps

    romix authored
  2. Support serialization of enums; better support serialization of arrays;

    eedrls authored
    increase the speed of serialization by 80-100 times!
This page is out of date. Refresh to see the latest.
View
395 src/main/java/net/kotek/jdbm/SerialClassInfo.java
@@ -7,7 +7,9 @@
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
/**
* This class stores information about serialized classes and fields.
@@ -27,7 +29,6 @@ public void serialize(DataOutput out, ArrayList<ClassInfo> obj) throws IOExcepti
out.writeUTF(fi.getType());
}
}
-
}
public ArrayList<ClassInfo> deserialize(DataInput in) throws IOException, ClassNotFoundException {
@@ -39,7 +40,7 @@ public void serialize(DataOutput out, ArrayList<ClassInfo> obj) throws IOExcepti
int fieldsNum = LongPacker.unpackInt(in);
FieldInfo[] fields = new FieldInfo[fieldsNum];
for (int j = 0; j < fieldsNum; j++) {
- fields[j] = new FieldInfo(in.readUTF(), in.readBoolean(), in.readUTF());
+ fields[j] = new FieldInfo(in.readUTF(), in.readBoolean(), in.readUTF(), Class.forName(className));
}
ret.add(new ClassInfo(className, fields));
}
@@ -64,11 +65,19 @@ public SerialClassInfo(DBAbstract db, long serialClassInfoRecid, ArrayList<Class
private final String name;
private final List<FieldInfo> fields = new ArrayList<FieldInfo>();
-
- ClassInfo(String name, FieldInfo[] fields) {
+ private final Map<String, FieldInfo> name2fieldInfo = new HashMap<String, FieldInfo>();
+ private final Map<String, Integer> name2fieldId = new HashMap<String, Integer>();
+ private ObjectStreamField[] objectStreamFields;
+ // Is this class an enumeration?
+ // 0 - if it is not an enum, 1 - if it is an enum, <0 if not initialized yet
+ private int isEnum = -1;
+
+ ClassInfo(String name, FieldInfo[] fields) {
this.name = name;
for (FieldInfo f : fields) {
+ this.name2fieldId.put(f.getName(), this.fields.size());
this.fields.add(f);
+ this.name2fieldInfo.put(f.getName(), f);
}
}
@@ -81,18 +90,13 @@ public String getName() {
}
public FieldInfo getField(String name) {
- for (FieldInfo field : fields) {
- if (field.getName().equals(name))
- return field;
- }
- return null;
+ return name2fieldInfo.get(name);
}
public int getFieldId(String name) {
- for (int i = 0; i < fields.size(); i++) {
- if (fields.get(i).getName().equals(name))
- return i;
- }
+ Integer fieldId = name2fieldId.get(name);
+ if(fieldId != null)
+ return fieldId;
return -1;
}
@@ -100,31 +104,134 @@ public FieldInfo getField(int serialId) {
return fields.get(serialId);
}
- public int addFieldInfo(FieldInfo field) {
- fields.add(field);
- return fields.size() - 1;
- }
+ public int addFieldInfo(FieldInfo field) {
+ name2fieldId.put(field.getName(), fields.size());
+ name2fieldInfo.put(field.getName(), field);
+ fields.add(field);
+ return fields.size() - 1;
+ }
+
+ public ObjectStreamField[] getObjectStreamFields() {
+ return objectStreamFields;
+ }
+ public void setObjectStreamFields(ObjectStreamField[] objectStreamFields) {
+ this.objectStreamFields = objectStreamFields;
+ }
+
+ public int getEnum() {
+ return isEnum;
+ }
+
+ public void setEnum(boolean isEnum) {
+ this.isEnum = isEnum?1:0;
+ }
}
/**
* Stores info about single field stored in JDBM.
* Roughly corresponds to 'java.io.ObjectFieldClass'
*/
- static class FieldInfo {
-
+ static class FieldInfo {
private final String name;
private final boolean primitive;
private final String type;
-
- public FieldInfo(String name, boolean primitive, String type) {
+ private Class typeClass;
+ // Class containing this field
+ private final Class clazz;
+ private Object setter;
+ private int setterIndex;
+ private Object getter;
+ private int getterIndex;
+
+ public FieldInfo(String name, boolean primitive, String type, Class clazz) {
this.name = name;
this.primitive = primitive;
this.type = type;
+ this.clazz = clazz;
+ try {
+ this.typeClass = Class.forName(type);
+ } catch (ClassNotFoundException e) {
+ this.typeClass = null;
+ }
+ initSetter();
+ initGetter();
}
- public FieldInfo(ObjectStreamField sf) {
- this(sf.getName(), sf.isPrimitive(), sf.getType().getName());
+ private void initSetter() {
+ // Set setter
+ String setterName = "set" + firstCharCap(name);
+ String fieldSetterName = clazz.getName() + "#" + setterName;
+
+ Class aClazz = clazz;
+
+ // iterate over class hierarchy, until root class
+ while (aClazz != Object.class) {
+ // check if there is getMethod
+ try {
+ Method m = aClazz.getMethod(setterName, typeClass);
+ if (m != null) {
+ setter = m;
+ return;
+ }
+ } catch (Exception e) {
+ // e.printStackTrace();
+ }
+
+ // no get method, access field directly
+ try {
+ Field f = aClazz.getDeclaredField(name);
+ // security manager may not be happy about this
+ if (!f.isAccessible())
+ f.setAccessible(true);
+ setter = f;
+ return;
+ } catch (Exception e) {
+// e.printStackTrace();
+ }
+ // move to superclass
+ aClazz = aClazz.getSuperclass();
+ }
+ }
+
+ private void initGetter() {
+ // Set setter
+ String getterName = "get" + firstCharCap(name);
+ String fieldSetterName = clazz.getName() + "#" + getterName;
+
+ Class aClazz = clazz;
+
+ // iterate over class hierarchy, until root class
+ while (aClazz != Object.class) {
+ // check if there is getMethod
+ try {
+ Method m = aClazz.getMethod(getterName);
+ if (m != null) {
+ getter = m;
+ return;
+ }
+ } catch (Exception e) {
+ // e.printStackTrace();
+ }
+
+ // no get method, access field directly
+ try {
+ Field f = aClazz.getDeclaredField(name);
+ // security manager may not be happy about this
+ if (!f.isAccessible())
+ f.setAccessible(true);
+ getter = f;
+ return;
+ } catch (Exception e) {
+// e.printStackTrace();
+ }
+ // move to superclass
+ aClazz = aClazz.getSuperclass();
+ }
+ }
+
+ public FieldInfo(ObjectStreamField sf, Class clazz) {
+ this(sf.getName(), sf.isPrimitive(), sf.getType().getName(), clazz);
}
public String getName() {
@@ -139,31 +246,38 @@ public String getType() {
return type;
}
+ private String firstCharCap(String s) {
+ return Character.toUpperCase(s.charAt(0)) + s.substring(1);
+ }
}
ArrayList<ClassInfo> registered;
+ Map<Class, Integer> class2classId = new HashMap<Class, Integer>();
+ Map<Integer, Class> classId2class = new HashMap<Integer, Class>();
final DBAbstract db;
public void registerClass(Class clazz) throws IOException {
- assertClassSerializable(clazz);
+ if(clazz != Object.class)
+ assertClassSerializable(clazz);
if (containsClass(clazz))
return;
-
-
ObjectStreamField[] streamFields = getFields(clazz);
FieldInfo[] fields = new FieldInfo[streamFields.length];
for (int i = 0; i < fields.length; i++) {
ObjectStreamField sf = streamFields[i];
- fields[i] = new FieldInfo(sf);
+ fields[i] = new FieldInfo(sf, clazz);
}
ClassInfo i = new ClassInfo(clazz.getName(), fields);
+ class2classId.put(clazz, registered.size());
+ classId2class.put(registered.size(), clazz);
registered.add(i);
+ i.setEnum(clazz.isEnum());
if (db != null)
db.update(serialClassInfoRecid, (Serialization) this, db.defaultSerializationSerializer);
@@ -171,103 +285,107 @@ public void registerClass(Class clazz) throws IOException {
}
private ObjectStreamField[] getFields(Class clazz) {
- ObjectStreamClass streamClass = ObjectStreamClass.lookup(clazz);
- FastArrayList<ObjectStreamField> fieldsList = new FastArrayList<ObjectStreamField>();
- while (streamClass != null) {
- for (ObjectStreamField f : streamClass.getFields()) {
- fieldsList.add(f);
- }
- clazz = clazz.getSuperclass();
- streamClass = ObjectStreamClass.lookup(clazz);
- }
- ObjectStreamField[] fields = new ObjectStreamField[fieldsList.size()];
- for (int i = 0; i < fields.length; i++) {
- fields[i] = fieldsList.get(i);
- }
+ ObjectStreamField[] fields = null;
+ ClassInfo classInfo = null;
+ Integer classId = class2classId.get(clazz);
+ if (classId != null) {
+ classInfo = registered.get(classId);
+ fields = classInfo.getObjectStreamFields();
+ }
+ if (fields == null) {
+ ObjectStreamClass streamClass = ObjectStreamClass.lookup(clazz);
+ FastArrayList<ObjectStreamField> fieldsList = new FastArrayList<ObjectStreamField>();
+ while (streamClass != null) {
+ for (ObjectStreamField f : streamClass.getFields()) {
+ fieldsList.add(f);
+ }
+ clazz = clazz.getSuperclass();
+ streamClass = ObjectStreamClass.lookup(clazz);
+ }
+ fields = new ObjectStreamField[fieldsList
+ .size()];
+ for (int i = 0; i < fields.length; i++) {
+ fields[i] = fieldsList.get(i);
+ }
+ if(classInfo != null)
+ classInfo.setObjectStreamFields(fields);
+ }
return fields;
}
private void assertClassSerializable(Class clazz) throws NotSerializableException, InvalidClassException {
+ if(containsClass(clazz))
+ return;
+
if (!Serializable.class.isAssignableFrom(clazz))
throw new NotSerializableException(clazz.getName());
}
-
- public Object getFieldValue(String fieldName, Object object) {
- Class clazz = object.getClass();
- //iterate over class hierarchy, until root class
- while (clazz != Object.class) {
- //check if there is getMethod
- try {
- Method m = clazz.getDeclaredMethod("get" + firstCharCap(fieldName));
- if (m != null)
- return m.invoke(object);
- } catch (Exception e) {
- // e.printStackTrace();
- }
- //no get method, access field directly
- try {
- Field f = clazz.getDeclaredField(fieldName);
- if (!f.isAccessible())
- f.setAccessible(true); // security manager may not be happy about this
- return f.get(object);
- } catch (Exception e) {
- // e.printStackTrace();
- }
- //move to superclass
- clazz = clazz.getSuperclass();
- }
- throw new NoSuchFieldError(object.getClass() + "." + fieldName);
- }
-
- public void setFieldValue(String fieldName, Object object, Object value) {
- Class clazz = object.getClass();
- //iterate over class hierarchy, until root class
- while (clazz != Object.class) {
- //check if there is getMethod
- try {
- Method m = clazz.getMethod("set" + firstCharCap(fieldName), value.getClass());
- if (m != null) {
- m.invoke(object, value);
- return;
- }
- } catch (Exception e) {
- // e.printStackTrace();
- }
- //no get method, access field directly
- try {
- Field f = clazz.getDeclaredField(fieldName);
- if (!f.isAccessible())
- f.setAccessible(true); // security manager may not be happy about this
- f.set(object, value);
- return;
- } catch (Exception e) {
- // e.printStackTrace();
- }
- //move to superclass
- clazz = clazz.getSuperclass();
- }
- throw new NoSuchFieldError(object.getClass() + "." + fieldName);
-
- }
-
-
- private String firstCharCap(String s) {
- return Character.toUpperCase(s.charAt(0)) + s.substring(1);
- }
-
+ public Object getFieldValue(String fieldName, Object object) {
+ try {
+ registerClass(object.getClass());
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ ClassInfo classInfo = registered.get(class2classId.get(object.getClass()));
+ return getFieldValue(classInfo.getField(fieldName), object);
+ }
+
+ public Object getFieldValue(FieldInfo fieldInfo, Object object) {
+
+ Object fieldAccessor = fieldInfo.getter;
+ try {
+ if (fieldAccessor instanceof Method) {
+ Method m = (Method) fieldAccessor;
+ return m.invoke(object);
+ } else {
+ Field f = (Field) fieldAccessor;
+ return f.get(object);
+ }
+ } catch (Exception e) {
+
+ }
+
+ throw new NoSuchFieldError(object.getClass() + "." + fieldInfo.getName());
+ }
+
+ public void setFieldValue(String fieldName, Object object, Object value) {
+ try {
+ registerClass(object.getClass());
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ ClassInfo classInfo = registered.get(class2classId.get(object.getClass()));
+ setFieldValue(classInfo.getField(fieldName), object, value);
+ }
+
+ public void setFieldValue(FieldInfo fieldInfo, Object object, Object value) {
+
+ Object fieldAccessor = fieldInfo.setter;
+ try {
+ if (fieldAccessor instanceof Method) {
+ Method m = (Method) fieldAccessor;
+ m.invoke(object, value);
+ } else {
+ Field f = (Field) fieldAccessor;
+ f.set(object, value);
+ }
+ return;
+ } catch (Throwable e) {
+ e.printStackTrace();
+ }
+
+ throw new NoSuchFieldError(object.getClass() + "." + fieldInfo.getName());
+ }
+
public boolean containsClass(Class clazz) {
- for (ClassInfo c : registered) {
- if (c.getName().equals(clazz.getName()))
- return true;
- }
- return false;
+ return (class2classId.get(clazz) != null);
}
public int getClassId(Class clazz) {
- for (int i = 0; i < registered.size(); i++) {
- if (registered.get(i).getName().equals(clazz.getName()))
- return i;
+ Integer classId = class2classId.get(clazz);
+ if(classId != null) {
+ return classId;
}
throw new Error("Class is not registered: " + clazz);
}
@@ -281,6 +399,11 @@ public void writeObject(DataOutput out, Object obj, FastArrayList objectStack) t
ClassInfo classInfo = registered.get(classId);
ObjectStreamField[] fields = getFields(obj.getClass());
+
+ if(classInfo.getEnum() > 0) {
+ int ordinal = ((Enum)obj).ordinal();
+ LongPacker.packInt(out, ordinal);
+ }
LongPacker.packInt(out, fields.length);
@@ -290,16 +413,14 @@ public void writeObject(DataOutput out, Object obj, FastArrayList objectStack) t
if (fieldId == -1) {
//field does not exists in class definition stored in db,
//propably new field was added so add field descriptor
- fieldId = classInfo.addFieldInfo(new FieldInfo(f));
+ fieldId = classInfo.addFieldInfo(new FieldInfo(f, obj.getClass()));
db.update(serialClassInfoRecid, (Serialization) this, db.defaultSerializationSerializer);
}
LongPacker.packInt(out, fieldId);
//and write value
- Object fieldValue = getFieldValue(f.getName(), obj);
+ Object fieldValue = getFieldValue(classInfo.getField(fieldId), obj);
serialize(out, fieldValue, objectStack);
}
-
-
}
@@ -308,17 +429,31 @@ public Object readObject(DataInput in, FastArrayList objectStack) throws IOExcep
try {
int classId = LongPacker.unpackInt(in);
ClassInfo classInfo = registered.get(classId);
- Class clazz = Class.forName(classInfo.getName());
+// Class clazz = Class.forName(classInfo.getName());
+ Class clazz = classId2class.get(classId);
+ if(clazz == null)
+ clazz = Class.forName(classInfo.getName());
assertClassSerializable(clazz);
- Object o = createInstance(clazz, Object.class);
+ Object o;
+
+ if(classInfo.getEnum() > 0) {
+ int ordinal = LongPacker.unpackInt(in);
+ o = clazz.getEnumConstants()[ordinal];
+ }
+ else {
+ o = createInstance(clazz, Object.class);
+ }
+
objectStack.add(o);
+
+
int fieldCount = LongPacker.unpackInt(in);
for (int i = 0; i < fieldCount; i++) {
int fieldId = LongPacker.unpackInt(in);
FieldInfo f = classInfo.getField(fieldId);
Object fieldValue = deserialize(in, objectStack);
- setFieldValue(f.getName(), o, fieldValue);
+ setFieldValue(f, o, fieldValue);
}
return o;
} catch (Exception e) {
@@ -326,6 +461,12 @@ public Object readObject(DataInput in, FastArrayList objectStack) throws IOExcep
}
}
+ //TODO dependecy on nonpublic JVM API
+ static private sun.reflect.ReflectionFactory rf =
+ sun.reflect.ReflectionFactory.getReflectionFactory();
+
+ private static Map<Class, Constructor> class2constuctor = new HashMap<Class, Constructor>();
+
/**
* Little trick to create new instance without using constructor.
* Taken from http://www.javaspecialists.eu/archive/Issue175.html
@@ -333,13 +474,15 @@ public Object readObject(DataInput in, FastArrayList objectStack) throws IOExcep
private static <T> T createInstance(Class<T> clazz, Class<? super T> parent) {
try {
- //TODO dependecy on nonpublic JVM API
- sun.reflect.ReflectionFactory rf =
- sun.reflect.ReflectionFactory.getReflectionFactory();
- Constructor objDef = parent.getDeclaredConstructor();
- Constructor intConstr = rf.newConstructorForSerialization(
- clazz, objDef
- );
+ Constructor intConstr = class2constuctor.get(clazz);
+
+ if (intConstr == null) {
+ Constructor objDef = parent.getDeclaredConstructor();
+ intConstr = rf.newConstructorForSerialization(
+ clazz, objDef);
+ class2constuctor.put(clazz, intConstr);
+ }
+
return clazz.cast(intConstr.newInstance());
} catch (RuntimeException e) {
throw e;
View
39 src/main/java/net/kotek/jdbm/Serialization.java
@@ -16,10 +16,13 @@
package net.kotek.jdbm;
import java.io.*;
+import java.lang.reflect.Array;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.*;
+import net.kotek.jdbm.SerialClassInfo.ClassInfo;
+
import static net.kotek.jdbm.SerializationHeader.*;
/**
@@ -37,7 +40,7 @@
* @author Jan Kotek
*/
@SuppressWarnings("unchecked")
-class Serialization extends SerialClassInfo implements Serializer {
+public class Serialization extends SerialClassInfo implements Serializer {
/**
@@ -54,12 +57,14 @@
public Serialization() {
super(null,0L,new ArrayList<ClassInfo>());
+ // Add java.lang.Object as registered class
+ registered.add(new ClassInfo(Object.class.getName(), new FieldInfo[]{}));
}
/**
* Serialize the object into a byte array.
*/
- byte[] serialize(Object obj)
+ public byte[] serialize(Object obj)
throws IOException {
ByteArrayOutputStream ba = new ByteArrayOutputStream();
DataOutputStream da = new DataOutputStream(ba);
@@ -310,6 +315,14 @@ else if (val == 1)
} else {
out.write(ARRAY_OBJECT);
LongPacker.packInt(out, b.length);
+
+ // Write class id for components
+ Class<?> componentType = obj.getClass().getComponentType();
+ registerClass(componentType);
+ //write class header
+ int classId = getClassId(componentType);
+ LongPacker.packInt(out, classId);
+
for (Object o : b)
serialize(out, o, objectStack);
@@ -366,6 +379,8 @@ else if (val == 1)
}
} else if (clazz == HashMap.class) {
serializeMap(HASHMAP, out, obj, objectStack);
+ } else if (clazz == IdentityHashMap.class) {
+ serializeMap(IDENTITYHASHMAP, out, obj, objectStack);
} else if (clazz == LinkedHashMap.class) {
serializeMap(LINKEDHASHMAP, out, obj, objectStack);
} else if (clazz == Hashtable.class) {
@@ -572,7 +587,7 @@ else if (val > 0 && val < 255) {
* @throws IOException
* @throws ClassNotFoundException
*/
- Object deserialize(byte[] buf) throws ClassNotFoundException, IOException {
+ public Object deserialize(byte[] buf) throws ClassNotFoundException, IOException {
ByteArrayInputStream bs = new ByteArrayInputStream(buf);
DataInputStream das = new DataInputStream(bs);
Object ret = deserialize(das);
@@ -913,6 +928,9 @@ public Object deserialize(DataInput is, FastArrayList objectStack) throws IOExce
case HASHMAP:
ret = deserializeHashMap(is, objectStack);
break;
+ case IDENTITYHASHMAP:
+ ret = deserializeIdentityHashMap(is, objectStack);
+ break;
case LINKEDHASHMAP:
ret = deserializeLinkedHashMap(is, objectStack);
break;
@@ -1062,7 +1080,11 @@ private Class deserializeClass(DataInput is) throws IOException, ClassNotFoundEx
private Object[] deserializeArrayObject(DataInput is, FastArrayList objectStack) throws IOException, ClassNotFoundException {
int size = LongPacker.unpackInt(is);
- Object[] s = new Object[size];
+ // Read class id for components
+ int classId = LongPacker.unpackInt(is);
+ Class clazz = classId2class.get(classId);
+
+ Object[] s = (Object[])Array.newInstance(clazz, size);
objectStack.add(s);
for (int i = 0; i < size; i++)
s[i] = deserialize(is, objectStack);
@@ -1188,6 +1210,15 @@ private Class deserializeClass(DataInput is) throws IOException, ClassNotFoundEx
return s;
}
+ private IdentityHashMap<Object, Object> deserializeIdentityHashMap(DataInput is, FastArrayList objectStack) throws IOException, ClassNotFoundException {
+ int size = LongPacker.unpackInt(is);
+
+ IdentityHashMap<Object, Object> s = new IdentityHashMap<Object, Object>(size);
+ objectStack.add(s);
+ for (int i = 0; i < size; i++)
+ s.put(deserialize(is, objectStack), deserialize(is, objectStack));
+ return s;
+ }
private LinkedHashMap<Object, Object> deserializeLinkedHashMap(DataInput is, FastArrayList objectStack) throws IOException, ClassNotFoundException {
int size = LongPacker.unpackInt(is);
View
3  src/main/java/net/kotek/jdbm/SerializationHeader.java
@@ -122,6 +122,9 @@
final static int DATE = 127;
+ final static int NOTUSED_IDENTITYHASHMAP = 128;
+ final static int IDENTITYHASHMAP = 129;
+
static final int JDBMLINKEDLIST = 159;
static final int HTREE = 160;
Something went wrong with that request. Please try again.