Rework stackoverflow

Signed-off-by: Jorge Bescos Gascon <jorge.bescos.gascon@oracle.com>

jaxb-ri/runtime/impl/src/main/java/org/glassfish/jaxb/runtime/v2/schemagen/XmlSchemaGenerator.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1997, 2022 Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2023 Oracle and/or its affiliates. All rights reserved.
  *
  * This program and the accompanying materials are made available under the
  * terms of the Eclipse Distribution License v. 1.0, which is available at
@@ -103,7 +103,7 @@ public final class XmlSchemaGenerator<T,C,F,M> {
     private final NonElement<T,C> anyType;
 
     /**
-     * Used to detect cycles in anonymous types.
+     * Used to detect cycles in types.
      */
     private final CollisionCheckStack<ClassInfo<T,C>> collisionChecker = new CollisionCheckStack<>();
 
@@ -405,8 +405,6 @@ public void write(SchemaOutputResolver resolver, ErrorListener errorListener) th
                     systemIds.put(n,output.getSystemId());
                 }
             }
-            //Clear the namespace specific set with already written classes
-            n.resetWritten();
         }
 
         // then write'em all
@@ -484,24 +482,12 @@ private class Namespace {
          */
         private boolean useMimeNs;
 
-        /**
-         * Container for already processed classes
-         */
-        private final Set<ClassInfo> written = new HashSet<>();
-
         public Namespace(String uri) {
             this.uri = uri;
             assert !XmlSchemaGenerator.this.namespaces.containsKey(uri);
             XmlSchemaGenerator.this.namespaces.put(uri,this);
         }
 
-        /**
-         * Clear out the set of already processed classes for this namespace
-         */
-        void resetWritten() {
-            written.clear();
-        }
-
         /**
          * Process the given PropertyInfo looking for references to namespaces that
          * are foreign to the given namespace.  Any foreign namespace references
@@ -742,29 +728,19 @@ private void writeTypeRef(TypeHost th, NonElement<T,C> type, String refAttName)
                 e = (Element)type;
             }
             if (type.getTypeName()==null) {
-                if ((e != null) && (e.getElementName() != null)) {
-                    th.block(); // so that the caller may write other attributes
-                    if(type instanceof ClassInfo) {
-                        writeClass( (ClassInfo<T,C>)type, th );
+                th.block(); // so that the caller may write other attributes
+                if(type instanceof ClassInfo) {
+                    if(collisionChecker.push((ClassInfo<T,C>)type)) {
+                        errorListener.warning(new SAXParseException(
+                            Messages.ANONYMOUS_TYPE_CYCLE.format(collisionChecker.getCycleString()),
+                            null
+                        ));
                     } else {
-                        writeEnum( (EnumLeafInfo<T,C>)type, (SimpleTypeHost)th);
+                        writeClass( (ClassInfo<T,C>)type, th );
                     }
+                    collisionChecker.pop();
                 } else {
-                    // anonymous
-                    th.block(); // so that the caller may write other attributes
-                    if(type instanceof ClassInfo) {
-                        if(collisionChecker.push((ClassInfo<T,C>)type)) {
-                            errorListener.warning(new SAXParseException(
-                                Messages.ANONYMOUS_TYPE_CYCLE.format(collisionChecker.getCycleString()),
-                                null
-                            ));
-                        } else {
-                            writeClass( (ClassInfo<T,C>)type, th );
-                        }
-                        collisionChecker.pop();
-                    } else {
-                        writeEnum( (EnumLeafInfo<T,C>)type, (SimpleTypeHost)th);
-                    }
+                    writeEnum( (EnumLeafInfo<T,C>)type, (SimpleTypeHost)th);
                 }
             } else {
                 th._attribute(refAttName,type.getTypeName());
@@ -807,10 +783,6 @@ private void writeEnum(EnumLeafInfo<T, C> e, SimpleTypeHost th) {
          * @param parent the writer of the parent element into which the type will be defined
          */
         private void writeClass(ClassInfo<T,C> c, TypeHost parent) {
-            if (written.contains(c)) { // to avoid cycles let's check if we haven't already processed the class
-                return;
-            }
-            written.add(c);
             // special handling for value properties
             if (containsValueProp(c)) {
                 if (c.getProperties().size() == 1) {

jaxb-ri/runtime/impl/src/test/java/org/glassfish/jaxb/runtime/v2/schemagen/Foo.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2023 Oracle and/or its affiliates. All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Distribution License v. 1.0, which is available at
+ * http://www.eclipse.org/org/documents/edl-v10.php.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+package org.glassfish.jaxb.runtime.v2.schemagen;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import jakarta.xml.bind.annotation.XmlAccessType;
+import jakarta.xml.bind.annotation.XmlAccessorType;
+import jakarta.xml.bind.annotation.XmlRootElement;
+import jakarta.xml.bind.annotation.XmlType;
+
+@XmlAccessorType(XmlAccessType.FIELD)
+@XmlType(name = "", propOrder = {"fooObject"})
+@XmlRootElement(name = "Foo")
+public class Foo {
+    protected List<Foo> fooObject;
+
+    public List<Foo> getFooObject() {
+        if (fooObject == null) {
+            fooObject = new ArrayList<Foo>();
+        }
+        return this.fooObject;
+    }
+}

jaxb-ri/runtime/impl/src/test/java/org/glassfish/jaxb/runtime/v2/schemagen/XmlSchemaGeneratorTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1997, 2022 Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2023 Oracle and/or its affiliates. All rights reserved.
  *
  * This program and the accompanying materials are made available under the
  * terms of the Eclipse Distribution License v. 1.0, which is available at
@@ -16,11 +16,15 @@
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.PrintWriter;
+import java.io.StringWriter;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.util.HashMap;
+import java.util.Map;
 
 import jakarta.xml.bind.JAXBContext;
 import jakarta.xml.bind.SchemaOutputResolver;
+import jakarta.xml.bind.MarshalException;
 import javax.xml.transform.Result;
 import javax.xml.transform.stream.StreamResult;
 
@@ -31,7 +35,21 @@ public class XmlSchemaGeneratorTest extends TestCase {
     public static void main(String[] args) {
         TestRunner.run(XmlSchemaGeneratorTest.class);
     }
-
+
+    public void testStackoverflow() throws Exception {
+        Map<String, Object> properties = new HashMap<>();
+        JAXBContext c = JAXBContext.newInstance(new Class[] {Foo.class}, properties);
+        Foo foo = new Foo();
+        foo.getFooObject().add(foo);
+        StringWriter sw = new StringWriter();
+        try {
+            c.createMarshaller().marshal(foo, sw);
+            fail("It is expected to catch MarshalException");
+        } catch (MarshalException e) {
+            // Expected
+        }
+    }
+
     public void test2() throws Exception {
         try {
             JAXBContext context = JAXBContext.newInstance(NSParent.class);