Fixed auto-creation for schema with circular dependencies

CHANGELOG

@@ -1,91 +1,94 @@
-CHANGELOG
-
-
-
-2013-Jun-09
-===========
-- T.zaeschke
-	- Schema writing
-	- New STATS: PAGE_CNT  
-
-2013-Mai
-===========
-- T.zaeschke
-	- XML reader/writer
-	- ZooCompareDb tools
-	- New disk page header with page type, page format version and tx id  
-
-2013-Jan/Feb
-===========
-- T.zaeschke
-	- Implemented schema versioning
-	- Implemented schema declaration without Java classes
-	- Implemented automatic schema evolution
-	- Implemented reading of non-Java instances
-	- Implemented writing of non-Java instances
-	- Index evolution  
-
-2013-Jan-xx
-===========
-- T.zaeschke
-	- Implemented JDO callback and synchronisation API  
-
-2012-Oct-29
-===========
-- T.Zaeschke: 
-    - Improved statistics collection.
-    - Improved commit() performance (separate caching for dirty/deleted).
-
-2012-Jun-24
-===========
-- T.Zaeschke: 
-    - Fixed queries with OR and multiple indices. All according test work now.
-
-2012-Jun-23
-===========
-- T.Zaeschke: 
-	- Finally implemented negation in queries. Still buggy though, see tests.
-
-2012-Jun-22
-===========
-- T.Zaeschke: 
-	- Fixed DBHasMap issues
-
-2012-Jun-20
-===========
-- T.Zaeschke: 
-	- Fixed Extents going over transaction boundaries
-
-2012-Jun-16
-===========
-- T.Zaeschke: 
-	- Fixed a number of issues with index updates, e.g. unique index collisions (Test_091)
-	- Implemented node-revert() for failed commits (eg index collision)
-
-2012-Jun-08
-===========
-- T.Zaeschke: 
-	- Schemas are now stored as normal objects. This fixes also all known page-leaks in the database
-	  for example when adding and removing schemas.
-	- Implemented serialization of enums.
-	- Fixed db-admin test
-
-2012-Jun-05
-===========
-- T.Zaeschke: 
-	- Started separating ZooDB from JDO, introduced ZooPCImpl
-
-2012-May-23
-===========
-- T.Zaeschke: 
-	- Fixed weird ConcurrentModificationException in deserializer properly
-	- Started pooling of de-serializers.
-
-
-2012-Apr-26
-===========
-- T.Zaeschke: Fixed weird ConcurrentModificationException in deserializer
-
-2012-Apr-??
-===========
+CHANGELOG
+
+2014-Feb-23
+===========
+- T.Zaeschke
+	- Proper testing and implementation of schema auto-creation
+
+2013-Jun-09
+===========
+- T.Zaeschke
+	- Schema writing
+	- New STATS: PAGE_CNT  
+
+2013-Mai
+===========
+- T.Zaeschke
+	- XML reader/writer
+	- ZooCompareDb tools
+	- New disk page header with page type, page format version and tx id  
+
+2013-Jan/Feb
+===========
+- T.Zaeschke
+	- Implemented schema versioning
+	- Implemented schema declaration without Java classes
+	- Implemented automatic schema evolution
+	- Implemented reading of non-Java instances
+	- Implemented writing of non-Java instances
+	- Index evolution  
+
+2013-Jan-xx
+===========
+- T.Zaeschke
+	- Implemented JDO callback and synchronisation API  
+
+2012-Oct-29
+===========
+- T.Zaeschke: 
+    - Improved statistics collection.
+    - Improved commit() performance (separate caching for dirty/deleted).
+
+2012-Jun-24
+===========
+- T.Zaeschke: 
+    - Fixed queries with OR and multiple indices. All according test work now.
+
+2012-Jun-23
+===========
+- T.Zaeschke: 
+	- Finally implemented negation in queries. Still buggy though, see tests.
+
+2012-Jun-22
+===========
+- T.Zaeschke: 
+	- Fixed DBHasMap issues
+
+2012-Jun-20
+===========
+- T.Zaeschke: 
+	- Fixed Extents going over transaction boundaries
+
+2012-Jun-16
+===========
+- T.Zaeschke: 
+	- Fixed a number of issues with index updates, e.g. unique index collisions (Test_091)
+	- Implemented node-revert() for failed commits (eg index collision)
+
+2012-Jun-08
+===========
+- T.Zaeschke: 
+	- Schemas are now stored as normal objects. This fixes also all known page-leaks in the database
+	  for example when adding and removing schemas.
+	- Implemented serialization of enums.
+	- Fixed db-admin test
+
+2012-Jun-05
+===========
+- T.Zaeschke: 
+	- Started separating ZooDB from JDO, introduced ZooPCImpl
+
+2012-May-23
+===========
+- T.Zaeschke: 
+	- Fixed weird ConcurrentModificationException in deserializer properly
+	- Started pooling of de-serializers.
+
+
+2012-Apr-26
+===========
+- T.Zaeschke: Fixed weird ConcurrentModificationException in deserializer
+
+2012-Apr-??
+===========
 - T.Zaeschke: Migrated to git repo

src/org/zoodb/jdo/PersistenceManagerImpl.java

@@ -82,7 +82,8 @@ public class PersistenceManagerImpl implements PersistenceManager {
      */
     PersistenceManagerImpl(PersistenceManagerFactoryImpl factory, String password) {
         this.factory = factory;
-    	nativeConnection = new Session(this, factory.getConnectionURL());
+    	nativeConnection = new Session(this, factory.getConnectionURL(),
+    			factory.getAutoCreateSchema());
         transaction = new TransactionImpl(this, 
         		factory.getRetainValues(),
         		factory.getOptimistic(),

src/org/zoodb/jdo/api/ZooSchema.java

@@ -48,7 +48,7 @@ private ZooSchema() {
 	public static ZooClass defineClass(PersistenceManager pm, Class<?> cls) {
     	checkValidity(pm);
 		Node node = Session.getSession(pm).getPrimaryNode();
-		return Session.getSession(pm).getSchemaManager().createSchema(node, cls, false);
+		return Session.getSession(pm).getSchemaManager().createSchema(node, cls);
 	}
 
 	/**

src/org/zoodb/jdo/internal/Session.java

@@ -71,10 +71,10 @@ public class Session implements IteratorRegistry {
 	private final WeakHashMap<CloseableIterator<?>, Object> extents = 
 	    new WeakHashMap<CloseableIterator<?>, Object>(); 
 
-	public Session(PersistenceManagerImpl pm, String dbPath) {
+	public Session(PersistenceManagerImpl pm, String dbPath, boolean autoCreateSchema) {
 		this.pm = pm;
 		this.cache = new ClientSessionCache(this);
-		this.schemaManager = new SchemaManager(cache);
+		this.schemaManager = new SchemaManager(cache, autoCreateSchema);
 		this.primary = ZooFactory.get().createNode(dbPath, cache);
 		this.nodes.add(primary);
 		this.cache.addNode(primary);

src/org/zoodb/jdo/internal/ZooClassDef.java

@@ -27,6 +27,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import org.zoodb.api.impl.ZooPCImpl;
 import org.zoodb.jdo.internal.ZooFieldDef.JdoType;
@@ -104,6 +105,7 @@ public static ZooClassDef bootstrapZooPCImpl() {
 		ZooClassDef x = new ZooClassDef(ZooPCImpl.class.getName(), 50, 0, 50, 0);
 		x.cls = ZooPCImpl.class;
 		x.className = ZooPCImpl.class.getName();
+		//x.associateFields(); //doesn't seem to be necessary
 		return x;
 	}
 
@@ -132,6 +134,9 @@ public static ZooClassDef bootstrapZooClassDef() {
 		meta.registerFields(fields);
 		meta.cls = ZooClassDef.class;
 		meta.className = ZooClassDef.class.getName();
+
+		meta.associateFields();
+		meta.associateJavaTypes();
 		return meta;
 	}
 
@@ -336,7 +341,8 @@ void registerFields(List<ZooFieldDef> fieldList) {
         localFields.addAll(fieldList);
     }
 
-    public void associateFCOs(Collection<ZooClassDef> cachedSchemata) {
+    public void associateFCOs(Collection<ZooClassDef> cachedSchemata, 
+    		boolean isSchemaAutoCreateMode, Set<String> missingSchemas) {
 		//Fields:
 		for (ZooFieldDef zField: localFields) {
 			if (zField.isPrimitiveType()) {
@@ -367,6 +373,10 @@ public void associateFCOs(Collection<ZooClassDef> cachedSchemata) {
 
 			if (typeDef == null) {
 				if (zField.getJdoType() == JdoType.REFERENCE) {
+					if (isSchemaAutoCreateMode) {
+						missingSchemas.add(zField.getTypeName());
+						continue;
+					}
 					String typeName = zField.getTypeName();
 					throw DBLogger.newUser("Schema error, class " + getClassName() + " references "
 							+ "class " + typeName + " as embedded object, but embedded objects "

src/org/zoodb/jdo/internal/client/SchemaManager.java

@@ -22,6 +22,8 @@
 
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
 
 import org.zoodb.api.impl.ZooPCImpl;
 import org.zoodb.jdo.api.ZooClass;
@@ -43,9 +45,11 @@ public class SchemaManager {
 
 	private ClientSessionCache cache;
 	private final ArrayList<SchemaOperation> ops = new ArrayList<SchemaOperation>();
-
-	public SchemaManager(ClientSessionCache cache) {
+	private final boolean isSchemaAutoCreateMode;
+
+	public SchemaManager(ClientSessionCache cache, boolean isSchemaAutoCreateMode) {
 		this.cache = cache;
+		this.isSchemaAutoCreateMode = isSchemaAutoCreateMode;
 	}
 
 	public boolean isSchemaDefined(Class<?> cls, Node node) {
@@ -122,7 +126,7 @@ private ZooClassProxy getSchemaProxy(ZooClassDef def) {
         return def.getVersionProxy();
 	}
 
-	public ZooClassProxy createSchema(Node node, Class<?> cls, boolean autoCreateSchema) {
+	public ZooClassProxy createSchema(Node node, Class<?> cls) {
 		if (isSchemaDefined(cls, node)) {
 			throw DBLogger.newUser("Schema is already defined: " + cls.getName());
 		}
@@ -150,8 +154,8 @@ public ZooClassProxy createSchema(Node node, Class<?> cls, boolean autoCreateSch
 		if (cls != ZooPCImpl.class) {
 			Class<?> clsSuper = cls.getSuperclass();
 			ZooClassDef defSuper = locateClassDefinition(clsSuper, node);
-			if (defSuper == null && autoCreateSchema) {
-				defSuper = createSchema(node, clsSuper, autoCreateSchema).getSchemaDef();
+			if (defSuper == null && isSchemaAutoCreateMode) {
+				defSuper = createSchema(node, clsSuper).getSchemaDef();
 			}
 			def = ZooClassDef.createFromJavaType(cls, defSuper, node, cache.getSession()); 
 		} else {
@@ -219,12 +223,18 @@ public boolean isIndexUnique(ZooFieldDef f) {
 	public void commit() {
 		//If nothing changed, there is no need to verify anything!
 		if (!ops.isEmpty()) {
-			Collection<ZooClassDef> schemata = cache.getSchemata();
-			for (ZooClassDef cs: schemata) {
-				if (cs.jdoZooIsDeleted()) continue;
-				//check ALL classes, e.g. to find references to removed classes
-				checkSchemaFields(cs, schemata);
-			}
+			Set<String> missingSchemas = new HashSet<String>();
+			//loop until all schemas are auto-defined (if in auto-mode)
+			do {
+				missingSchemas.clear();
+				Collection<ZooClassDef> schemata = cache.getSchemata();
+				for (ZooClassDef cs: schemata) {
+					if (cs.jdoZooIsDeleted()) continue;
+					//check ALL classes, e.g. to find references to removed classes
+					checkSchemaFields(cs, schemata, missingSchemas);
+				}
+				addMissingSchemas(missingSchemas);
+			} while (!missingSchemas.isEmpty());
 		}
 
 		// perform pending operations
@@ -236,17 +246,40 @@ public void commit() {
 		ops.clear();
 	}
 
+	/**
+	 * This method add all schemata that were found missing when checking all known
+	 * schemata.
+	 * @param missingSchemas
+	 */
+	private void addMissingSchemas(Set<String> missingSchemas) {
+		if (missingSchemas.isEmpty()) {
+			return;
+		}
+		for (String className: missingSchemas) {
+			Class<?> cls;
+			try {
+				cls = Class.forName(className);
+			} catch (ClassNotFoundException e) {
+				throw DBLogger.newFatal("Invalid field type in schema", e);
+			}
+			//TODO primary node is not always right here...
+			createSchema(cache.getSession().getPrimaryNode(), cls);
+		}
+	}
+
 	/**
 	 * Check the fields defined in this class.
 	 * @param schema
+	 * @param missingSchemas 
 	 * @param schemata 
 	 */
-	private void checkSchemaFields(ZooClassDef schema, Collection<ZooClassDef> cachedSchemata) {
+	private void checkSchemaFields(ZooClassDef schema, Collection<ZooClassDef> cachedSchemata, 
+			Set<String> missingSchemas) {
 		//do this only now, because only now we can check which field types
 		//are really persistent!
 		//TODO check for field types that became persistent only now -> error!!
 		//--> requires schema evolution.
-		schema.associateFCOs(cachedSchemata);
+		schema.associateFCOs(cachedSchemata, isSchemaAutoCreateMode, missingSchemas);
 
 //		TODO:
 //			- construct fieldDefs here an give them to classDef.
@@ -338,4 +371,8 @@ public void renameField(ZooFieldDef field, String fieldName) {
 		ops.add(new SchemaOperation.SchemaFieldRename(field, fieldName));
 		def.jdoZooMarkDirty();
 	}
+
+	public boolean getAutoCreateSchema() {
+		return isSchemaAutoCreateMode;
+	}
 }

src/org/zoodb/jdo/internal/client/session/ClientSessionCache.java

@@ -78,8 +78,6 @@ public ClientSessionCache(Session session) {
 		this.session = session;
 		ZooClassDef zpc = ZooClassDef.bootstrapZooPCImpl();
 		metaSchema = ZooClassDef.bootstrapZooClassDef();
-		metaSchema.associateFields();
-		metaSchema.associateJavaTypes();
 		metaSchema.initProvidedContext(session, session.getPrimaryNode());
 		schemata.put(zpc.getOid(), zpc);
 		schemata.put(metaSchema.getOid(), metaSchema);