fix hessian2 serialization infinit recursion(StackOverflowError) when…

… object's writeReplace method returns the object itself. (#1)

src/main/java/com/alibaba/com/caucho/hessian/io/JavaSerializer.java

@@ -219,13 +219,17 @@ public void writeObject(Object obj, AbstractHessianOutput out)
                 else
                     repl = _writeReplace.invoke(obj);
 
-                out.removeRef(obj);
+                //Some class would return itself for wrapReplace, which would cause infinite recursion
+                //In this case, we could write the object just like normal cases
+                if (repl != obj) {
+                    out.removeRef(obj);
 
-                out.writeObject(repl);
+                    out.writeObject(repl);
 
-                out.replaceRef(repl, obj);
+                    out.replaceRef(repl, obj);
 
-                return;
+                    return;
+                }
             }
         } catch (RuntimeException e) {
             throw e;

src/test/java/com/alibaba/com/caucho/hessian/io/writereplace/Hessian2WriteReplaceTest.java

@@ -0,0 +1,114 @@
+package com.alibaba.com.caucho.hessian.io.writereplace;
+
+import static org.junit.Assert.assertEquals;
+
+import com.alibaba.com.caucho.hessian.io.base.SerializeTestBase;
+import java.io.Serializable;
+import org.junit.Test;
+
+
+public class Hessian2WriteReplaceTest extends SerializeTestBase {
+
+  @Test
+  public void testWriteReplaceReturningItself() throws Exception {
+    String someName = "some name";
+    WriteReplaceReturningItself object = new WriteReplaceReturningItself(someName);
+
+    WriteReplaceReturningItself result = baseHessian2Serialize(object);
+
+    assertEquals(someName, result.getName());
+  }
+
+  @Test
+  public void testNormalWriteReplace() throws Exception {
+    String someFirstName = "first";
+    String someLastName = "last";
+    String someAddress = "some address";
+
+    NormalWriteReplace object = new NormalWriteReplace(someFirstName, someLastName, someAddress);
+
+    NormalWriteReplace result = baseHessian2Serialize(object);
+
+    assertEquals(someFirstName, result.getFirstName());
+    assertEquals(someLastName, result.getLastName());
+    assertEquals(someAddress, result.getAddress());
+  }
+
+  static class WriteReplaceReturningItself implements Serializable {
+
+    private static final long serialVersionUID = 1L;
+
+    private String name;
+
+    WriteReplaceReturningItself(String name) {
+      this.name = name;
+    }
+
+    public String getName() {
+      return name;
+    }
+
+    /**
+     * Some object may return itself for wrapReplace, e.g.
+     * https://github.com/FasterXML/jackson-databind/blob/master/src/main/java/com/fasterxml/jackson/databind/JsonMappingException.java#L173
+     */
+    Object writeReplace() {
+      //do some extra things
+
+      return this;
+    }
+  }
+
+  static class NormalWriteReplace implements Serializable {
+    private static final long serialVersionUID = 1L;
+
+    private String firstName;
+    private String lastName;
+    private String address;
+
+    public NormalWriteReplace(String firstName, String lastName, String address) {
+      this.firstName = firstName;
+      this.lastName = lastName;
+      this.address = address;
+    }
+
+    public String getFirstName() {
+      return firstName;
+    }
+
+    public String getLastName() {
+      return lastName;
+    }
+
+    public String getAddress() {
+      return address;
+    }
+
+    //return a proxy to save space for serialization
+    Object writeReplace() {
+      return new NormalWriteReplaceProxy(this);
+    }
+  }
+
+  static class NormalWriteReplaceProxy implements Serializable {
+    private static final long serialVersionUID = 1L;
+
+    private String data;
+
+    //empty constructor for deserialization
+    public NormalWriteReplaceProxy() {
+    }
+
+    public NormalWriteReplaceProxy(NormalWriteReplace normalWriteReplace) {
+      this.data = normalWriteReplace.getFirstName() + "," + normalWriteReplace.getLastName() + "," + normalWriteReplace
+          .getAddress();
+    }
+
+    //construct the actual object
+    Object readResolve() {
+      String[] parts = data.split(",");
+
+      return new NormalWriteReplace(parts[0], parts[1], parts[2]);
+    }
+  }
+}