if a bulk insert is too big for 1 buffer, break it up JAVA-66

mongodb · Dec 21, 2009 · 76ccc51 · 76ccc51
1 parent e69e2e5
commit 76ccc51
Show file tree

Hide file tree

Showing 3 changed files with 73 additions and 19 deletions.
diff --git a/src/main/com/mongodb/ByteEncoder.java b/src/main/com/mongodb/ByteEncoder.java
@@ -172,14 +172,19 @@ public int putObject( DBObject o ){
         }
         catch ( BufferOverflowException bof ){
             reset();
-            throw new IllegalArgumentException( "tried to save too large of an object.  max size : " + ( _buf.capacity() / 2  ) );
+            throw getTooLargeException();
         }
     }
 
+    RuntimeException getTooLargeException(){
+        return new IllegalArgumentException( "tried to save too large of an object.  " + 
+                                             " max size : " + ( _buf.capacity() / 2  ) );
+    }
+
     /**
      * this is really for embedded objects
      */
-    private int putObject( String name , DBObject o ){
+    int putObject( String name , DBObject o ){
         if ( o == null )
             throw new NullPointerException( "can't save a null object" );
 

diff --git a/src/main/com/mongodb/DBApiLayer.java b/src/main/com/mongodb/DBApiLayer.java
@@ -181,27 +181,42 @@ protected DBObject[] insert(DBObject[] arr, boolean shouldApply )
                         ((ObjectId)id)._new = false;
                 }
             }
-
-            ByteEncoder encoder = ByteEncoder.get();
-
-            encoder._buf.putInt( 0 ); // reserved
-            encoder._put( _fullNameSpace );
-
-            for (DBObject o : arr) {
-                encoder.putObject( o );
-            }
-            encoder.flip();
-
-            try {
-                doInsert( encoder._buf , getWriteConcern() );
-            }
-            finally {
-                encoder.done();
+
+            int cur = 0;
+            while ( cur < arr.length ){
+                ByteEncoder encoder = ByteEncoder.get();
+
+                encoder._buf.putInt( 0 ); // reserved
+                encoder._put( _fullNameSpace );
+
+                int n=0;
+                for ( ; cur<arr.length; cur++ ){
+                    DBObject o = arr[cur];
+                    int pos = encoder._buf.position();
+                    try {
+                        encoder.putObject( null , o );
+                        n++;
+                    }
+                    catch ( BufferOverflowException e ){
+                        if ( n == 0 )
+                            throw encoder.getTooLargeException();
+                        encoder._buf.position( pos );
+                        break;
+                    }
+                }
+                encoder.flip();
+
+                try {
+                    doInsert( encoder._buf , getWriteConcern() );
+                }
+                finally {
+                    encoder.done();
+                }
             }
 
             return arr;
         }
-
+        
         public void remove( DBObject o )
             throws MongoException {
 

diff --git a/src/test/com/mongodb/JavaClientTest.java b/src/test/com/mongodb/JavaClientTest.java
@@ -441,6 +441,40 @@ public void testObjectIdCompat2(){
     }
 
 
+    @Test
+    public void testLargeBulkInsert(){
+        DBCollection c = _db.getCollection( "largebulk" );
+        c.drop();
+        String s = "asdasdasd";
+        while ( s.length() < 10000 )
+            s += s;
+        List<DBObject> l = new ArrayList<DBObject>();
+        final int num = 3 * ( Bytes.MAX_OBJECT_SIZE / s.length() );
+
+        for ( int i=0; i<num; i++ ){
+            l.add( BasicDBObjectBuilder.start()
+                   .add( "_id" , i )
+                   .add( "x" , s )
+                   .get() );
+        }
+        assertEquals( 0 , c.find().count() );
+        c.insert( l );
+        assertEquals( num , c.find().count() );
+
+        s = l.toString();
+        assertTrue( s.length() > Bytes.MAX_OBJECT_SIZE );
+
+        boolean worked = false;
+        try {
+            c.save( new BasicDBObject( "foo" , s ) );
+            worked = true;
+        }
+        catch ( IllegalArgumentException ie ){}
+        assertFalse( worked );
+
+        assertEquals( num , c.find().count() );
+    }
+
 
     final Mongo _mongo;
     final DB _db;