Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Integrated Casbah's Optimized Lazy Decoding; need to track down a

remaining bug in the actual decoding logic to finish testing.
  • Loading branch information...
commit b020e27e21922c29442a5d630c5baf2dbc29e696 1 parent 0b5068b
@bwmcadams bwmcadams authored
View
2  persistence/mongodb-record/src/main/scala/net/liftweb/mongodb/record/MongoMetaRecord.scala
@@ -64,7 +64,7 @@ trait MongoMetaRecord[BaseRecord <: MongoRecord[BaseRecord]]
}
protected def useColl[T](f: DBCollection => T) =
- MongoDB.useCollection(mongoIdentifier, collectionName)(f)
+ MongoDB.useCollection(mongoIdentifier, collectionName, true)(f) // Lazy read mode (TODO - Cache lazy collections?)
def bulkDelete_!!(qry: DBObject): Unit = {
useColl(coll =>
View
47 persistence/mongodb-record/src/main/scala/net/liftweb/mongodb/record/field/DBRefField.scala
@@ -28,6 +28,7 @@ import scala.xml.NodeSeq
import com.mongodb.{BasicDBObject, BasicDBObjectBuilder, DBObject, DBRef}
import com.mongodb.util.JSON
import org.bson.types.ObjectId
+import org.bson.BSONObject
/*
* Field for storing a DBRef
@@ -75,17 +76,55 @@ class DBRefField[OwnerType <: BsonRecord[OwnerType], RefType <: MongoRecord[RefT
case null => Full(set(null))
case s: String => setFromString(s)
case None | Empty | Failure(_, _, _) => Full(set(null))
- case o => setFromString(o.toString)
+ /**
+ * BWM - 8/19/11
+ * In some cases such as Lazy decoding MongoDB may return an
+ * instance of BSONObject instead of a concrete DBRef for performance reasons.
+ *
+ * If we didn't catch an actual DBRef above, but find a BSONObject, attempt to extract
+ * the reference related information from the reference to instantiate a new DBRef
+ * ( future releases of the java driver are expected to return DBRefs but this is
+ * a good safe fallback which performs better than serializing to JSON )
+ */
+ case dbo: BSONObject if dbo.containsField("$ref") && dbo.containsField("$id") =>
+ setRawRef(dbo.get("$ref").asInstanceOf[String], dbo.get("$id"))
+ case o => setFromString(o.toString) // TODO - Consider using com.mongodb.util.JSON.serialize instead of assuming toString creates JSON
}
+ /**
+ * Create and set a reference from 'raw' DBRef
+ * info that came either from JSON or a basic nested
+ * DBObject (as Lazy mode may produce)
+ *
+ * Caller is expected to have already converted the $id
+ * field to a valid type (such as ObjectId if appropriate)
+ *
+ * TODO - Shouldn't we be running an invariant check that
+ * $ref points to the collection we expect it to?!
+ */
+ def setRawRef(refTo: String, id: AnyRef): Box[DBRef] =
+ MongoDB.use(ref.meta.mongoIdentifier) { db =>
+ Full(set(new DBRef(db, refTo, id)))
+ }
+
+
// assume string is json
def setFromString(in: String): Box[DBRef] = {
- val dbo = JSON.parse(in).asInstanceOf[BasicDBObject]
+ /* BWM - 8/19/11
+ * We shouldn't ever assume things are BasicDBObject, which is
+ * a very specific concrete instance of DBObject. Treat it instead
+ * as a generic interface of BSONObject (since we don't need
+ * the DBObject partialObject hooks)
+ *
+ * It is entirely possible to get something other than BasicDBObject
+ * back which would cause this to blow up at runtime.
+ */
+ val dbo = JSON.parse(in).asInstanceOf[BSONObject]
MongoDB.use(ref.meta.mongoIdentifier) ( db => {
val id = dbo.get("$id").toString
ObjectId.isValid(id) match {
- case true => Full(set(new DBRef(db, dbo.get("$ref").toString, new ObjectId(id))))
- case false => Full(set(new DBRef(db, dbo.get("$ref").toString, id)))
+ case true => setRawRef(dbo.get("$ref").toString, new ObjectId(id))
+ case false => setRawRef(dbo.get("$ref").toString, id)
}
})
}
View
39 ...tence/mongodb-record/src/main/scala/net/liftweb/mongodb/record/field/MongoListField.scala
@@ -32,7 +32,8 @@ import net.liftweb.record.{Field, FieldHelpers, MandatoryTypedField, Record}
import util.Helpers.tryo
import com.mongodb._
-import org.bson.types.ObjectId
+import org.bson.types.{BasicBSONList, ObjectId}
+import org.bson.BSONObject
/**
* List field. Compatible with most object types,
@@ -104,13 +105,41 @@ class MongoListField[OwnerType <: BsonRecord[OwnerType], ListType](rec: OwnerTyp
dbl
}
- // set this field's value using a DBObject returned from Mongo.
- def setFromDBObject(dbo: DBObject): Box[List[ListType]] =
- setBox(Full(dbo.asInstanceOf[BasicDBList].toList.asInstanceOf[List[ListType]]))
+ /**
+ * set this field's value using a DBObject returned from Mongo.
+ *
+ * BWM - 8/19/11
+ * This unfortunately previously didn't really test invariants well,
+ * and assumes that, as we were expecting a List we *must* have
+ * gotten a DBObject that *is* a list.
+ *
+ * Of course if you try casting as a list and it isn't... BOOM
+ *
+ * Some cases such as a broken database may give us an actual non-List DBObject
+ * however. Also, Lazy mode may return an un-list'ed DBObject instead of a
+ * instance of DBList.
+ *
+ * For now, if it isn't an instance of BasicBSONList,
+ * I'm simply invoking toList on a BSONObject cast
+ *
+ * TODO - invariant testing for validity of a non DBList convert-to-List?
+ */
+ def setFromDBObject(dbo: DBObject): Box[List[ListType]] = setBox(dbo match {
+ case l: BasicBSONList => Full(l.toList.asInstanceOf[List[ListType]])
+ /* If it wasn't a list but a normal BSONObject, still try toList-ing
+ * TODO - Should we check that the keys are all integers to be sure?
+ */
+ case o: BSONObject =>
+ val b = List.newBuilder[ListType]
+ // TODO - I'm not sure I like this explicit blind cast to ListType
+ for (k <- o.keySet()) b += o.get(k).asInstanceOf[ListType]
+ Full(b.result)
+ case default => Failure("Unable to convert DBObject to a List: " + default.getClass)
+ })
}
/*
-* List of Dates. Use MongListField[OwnerType, Date] instead.
+* List of Dates. Use MongoListField[OwnerType, Date] instead.
*/
@deprecated("Use MongListField[OwnerType, Date] instead")
class MongoDateListField[OwnerType <: BsonRecord[OwnerType]](rec: OwnerType)
View
7 ...ce/mongodb-record/src/test/scala/net/liftweb/mongodb/record/MongoRecordExamplesSpec.scala
@@ -271,6 +271,7 @@ object MongoRecordExamplesSpec extends Specification("MongoRecordExamples Specif
fromDb.isDefined must_== true
+ var x = 0
for (t <- fromDb) {
t._id.value must_== tr._id.value
t.booleanfield.value must_== tr.booleanfield.value
@@ -278,6 +279,12 @@ object MongoRecordExamplesSpec extends Specification("MongoRecordExamples Specif
TstRecord.formats.dateFormat.format(tr.datetimefield.value.getTime)
t.doublefield.value must_== tr.doublefield.value
t.intfield.value must_== tr.intfield.value
+ x += 1
+ System.err.println("(%d) [ID: %s] T Locale: '0x%02X' [%d - %s / %s] TR Locale: '%s' [%d - %s / %s]".format(
+ x, t._id,
+ t.localefield.value.toString().getBytes()(5), t.localefield.value.length(), t.localefield.getClass, t.localefield.value.getClass,
+ tr.localefield, tr.localefield.value.length(), tr.localefield.getClass, t.localefield.value.getClass)
+ )
t.localefield.value must_== tr.localefield.value
t.longfield.value must_== tr.longfield.value
t.passwordfield.isMatch(pwd) must_== true
View
27 persistence/mongodb/src/main/scala/net/liftweb/mongodb/JObjectParser.scala
@@ -26,7 +26,8 @@ import net.liftweb.json._
import net.liftweb.common.Box
import com.mongodb.{BasicDBObject, BasicDBList, DBObject}
-import org.bson.types.ObjectId
+import org.bson.BSONObject
+import org.bson.types.{BasicBSONList, ObjectId}
object JObjectParser {
@@ -48,8 +49,28 @@ object JObjectParser {
case x if primitive_?(x.getClass) => primitive2jvalue(x)
case x if datetype_?(x.getClass) => datetype2jvalue(x)(formats)
case x if mongotype_?(x.getClass) => mongotype2jvalue(x)(formats)
- case x: BasicDBList => JArray(x.toList.map( x => serialize(x, formats)))
- case x: BasicDBObject => JObject(
+ /**
+ * BWM 8/19/11:
+ * This was using the concrete MongoDB "BasicDBList" class,
+ * moved it backwards to BasicBSONList to allow subclasses to work.
+ *
+ * TODO - There is no abstract base interface for DBList, but I'm of a mind
+ * that any Array or List-like structure should be parsed into JArray
+ */
+ case x: BasicBSONList => JArray(x.toList.map( x => serialize(x, formats)))
+
+ /**
+ * BWM - 8/19/11
+ * This formerly used a concrete BasicDBObject class, which breaks
+ * support for any custom DBObject implementations such as Casbah's
+ * MongoDBObject, Java Driver's LazyDBObject or any other customisation.
+ *
+ * Moved this back completely to the BSON Type "BSONObject" for the widest
+ * possible MongoDB support. All DBObjects inherit from BSONObject and the
+ * standard for most stuff at 10gen / MongoDB is to recommend filtering on
+ * BSONObject to allow for user customised types.
+ */
+ case x: BSONObject => JObject(
x.keySet.toList.map { f =>
JField(f.toString, serialize(x.get(f.toString), formats))
}
View
79 persistence/mongodb/src/main/scala/net/liftweb/mongodb/Mongo.scala
@@ -22,6 +22,7 @@ import java.util.concurrent.ConcurrentHashMap
import scala.collection.immutable.HashSet
import com.mongodb._
+import com.mongodb.casbah.util.bson.decoding.OptimizedLazyDBDecoder
/*
* A trait for identfying Mongo instances
@@ -134,8 +135,37 @@ object MongoDB {
/*
* Get a Mongo collection. Gets a Mongo db first.
*/
- private def getCollection(name: MongoIdentifier, collectionName: String): Option[DBCollection] = getDb(name) match {
- case Some(mongo) if mongo != null => Some(mongo.getCollection(collectionName))
+ private def getCollection(name: MongoIdentifier, collectionName: String): Option[DBCollection] =
+ getCollection(name, collectionName, false)
+
+ /*
+ * Get a Mongo collection. Gets a Mongo db first.
+ *
+ * Setting lazyReads will use a LazyBSON Decoder,
+ * which skips decoding the BSON until you ask for a specific
+ * field value.
+ *
+ * LazyReads are recommended for Lift-Record. They may or may not
+ * yield improvements for the Lift-MongoDB DSL, etc... the lazy reads
+ * are *NOT* memoized so YMMV in non-Record situations.
+ */
+ private def getCollection(name: MongoIdentifier, collectionName: String, lazyReads: Boolean): Option[DBCollection] = getDb(name) match {
+ case Some(mongo) if mongo != null =>
+ val coll = mongo.getCollection(collectionName)
+ if (lazyReads) {
+ /**
+ * Setup a *lazy* decoded connection.
+ * This should yield significant performance benefits for lift-mongodb-record.
+ * The BSON is not fully deserialized into a DBObject, instead it is read lazily
+ * as each field is requested. With Lift-MongoDB-Record this means there will
+ * be essentially *no* middle decoding step.
+ *
+ * Instead of MongoDB->BSON->DBObject->Lift-Record-Object we now go
+ * MongoDB->BSON->Lift-Record-Object.
+ */
+ coll.setDBDecoderFactory(OptimizedLazyDBDecoder.Factory)
+ }
+ Some(coll)
case _ => None
}
@@ -168,9 +198,25 @@ object MongoDB {
/**
* Executes function {@code f} with the mongo named {@code name} and collection names {@code collectionName}.
* Gets a collection for you.
+ *
*/
- def useCollection[T](name: MongoIdentifier, collectionName: String)(f: (DBCollection) => T): T = {
- val coll = getCollection(name, collectionName) match {
+ def useCollection[T](name: MongoIdentifier, collectionName: String)(f: (DBCollection) => T): T =
+ useCollection[T](name, collectionName, false)(f)
+
+ /**
+ * Executes function {@code f} with the mongo named {@code name} and collection names {@code collectionName}.
+ * Gets a collection for you.
+ *
+ * Setting lazyReads will use a LazyBSON Decoder,
+ * which skips decoding the BSON until you ask for a specific
+ * field value.
+ *
+ * LazyReads are recommended for Lift-Record. They may or may not
+ * yield improvements for the Lift-MongoDB DSL, etc... the lazy reads
+ * are *NOT* memoized so YMMV in non-Record situations.
+ */
+ def useCollection[T](name: MongoIdentifier, collectionName: String, lazyReads: Boolean)(f: (DBCollection) => T): T = {
+ val coll = getCollection(name, collectionName, lazyReads) match {
case Some(collection) => collection
case _ => throw new MongoException("Collection not found: "+collectionName+". MongoIdentifier: "+name.toString)
}
@@ -179,10 +225,27 @@ object MongoDB {
}
/**
- * Same as above except uses DefaultMongoIdentifier
- */
- def useCollection[T](collectionName: String)(f: (DBCollection) => T): T = {
- val coll = getCollection(DefaultMongoIdentifier, collectionName) match {
+ * Executes function {@code f} with the mongo named {@code name} and collection names {@code collectionName}.
+ * Gets a collection for you, using DefaultMongoIdentifier.
+ *
+ */
+ def useCollection[T](collectionName: String)(f: (DBCollection) => T): T =
+ useCollection[T](collectionName, false)(f)
+
+ /**
+ * Executes function {@code f} with the mongo named {@code name} and collection names {@code collectionName}.
+ * Gets a collection for you, using DefaultMongoIdentifier.
+ *
+ * Setting lazyReads will use a LazyBSON Decoder,
+ * which skips decoding the BSON until you ask for a specific
+ * field value.
+ *
+ * LazyReads are recommended for Lift-Record. They may or may not
+ * yield improvements for the Lift-MongoDB DSL, etc... the lazy reads
+ * are *NOT* memoized so YMMV in non-Record situations.
+ */
+ def useCollection[T](collectionName: String, lazyReads: Boolean)(f: (DBCollection) => T): T = {
+ val coll = getCollection(DefaultMongoIdentifier, collectionName, lazyReads) match {
case Some(collection) => collection
case _ => throw new MongoException("Collection not found: "+collectionName+". MongoIdentifier: "+DefaultMongoIdentifier.toString)
}
Please sign in to comment.
Something went wrong with that request. Please try again.