Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

SERVER-4593: Ensure mongoimport allows imports of up to 16MB in BSON …

…size.
  • Loading branch information...
commit 186721566a69192a91d75f424757b5edb2a6c257 1 parent 37b331b
Ben Becker authored
2  debian/mongoimport.1
View
@@ -82,7 +82,7 @@ make sure this is indexed
stop importing at first error rather than continuing
.TP
.B \-\-jsonArray
-load a json array, not one item per line. Currently limited to 4MB.
+load a json array, not one item per line. Currently limited to 16MB.
.SH "COPYRIGHT"
.PP
Copyright 2007\-2011 10gen
9 src/mongo/bson/util/builder.h
View
@@ -19,6 +19,8 @@
#include <cfloat>
#include <string>
+#include <sstream>
+#include <iostream>
#include <string.h>
#include <stdio.h>
#include "../inline_decls.h"
@@ -203,8 +205,11 @@ namespace mongo {
a = 512;
if ( l > a )
a = l + 16 * 1024;
- if ( a > BufferMaxSize )
- msgasserted(13548, "BufBuilder grow() > 64MB");
+ if ( a > BufferMaxSize ) {
+ std::stringstream ss;
+ ss << "BufBuilder attempted to grow() to " << a << " bytes, past the 64MB limit.";
+ msgasserted(13548, ss.str().c_str());
+ }
data = (char *) al.Realloc(data, a);
size= a;
}
16 src/mongo/db/json.cpp
View
@@ -44,15 +44,17 @@ namespace mongo {
struct ObjectBuilder : boost::noncopyable {
~ObjectBuilder() {
- unsigned i = builders.size();
- if ( i ) {
- i--;
- for ( ; i>=1; i-- ) {
- if ( builders[i] ) {
- builders[i]->done();
+ DESTRUCTOR_GUARD(
+ unsigned i = builders.size();
+ if ( i ) {
+ i--;
+ for ( ; i>=1; i-- ) {
+ if ( builders[i] ) {
+ builders[i]->done();
+ }
}
}
- }
+ );
}
BSONObjBuilder *back() {
return builders.back().get();
18 src/mongo/tools/import.cpp
View
@@ -30,6 +30,8 @@
#include <boost/algorithm/string.hpp>
using namespace mongo;
+using std::string;
+using std::stringstream;
namespace po = boost::program_options;
@@ -45,7 +47,7 @@ class Import : public Tool {
bool _doimport;
bool _jsonArray;
vector<string> _upsertFields;
- static const int BUF_SIZE = 1024 * 1024 * 4;
+ static const int BUF_SIZE = 1024 * 1024 * 16;
void csvTokenizeRow(const string& row, vector<string>& tokens) {
bool inQuotes = false;
@@ -146,7 +148,11 @@ class Import : public Tool {
return -1;
int jslen;
- o = fromjson(buf, &jslen);
+ try {
+ o = fromjson(buf, &jslen);
+ } catch ( MsgAssertionException& e ) {
+ uasserted(13293, string("BSON representation of supplied JSON array is too large: ") + e.what());
+ }
len += jslen;
return len;
@@ -176,7 +182,11 @@ class Import : public Tool {
*end = 0;
end--;
}
- o = fromjson( line );
+ try {
+ o = fromjson( line );
+ } catch ( MsgAssertionException& e ) {
+ uasserted(13504, string("BSON representation of supplied JSON is too large: ") + e.what());
+ }
return true;
}
@@ -259,7 +269,7 @@ class Import : public Tool {
("upsert", "insert or update objects that already exist" )
("upsertFields", po::value<string>(), "comma-separated fields for the query part of the upsert. You should make sure this is indexed" )
("stopOnError", "stop importing at first error rather than continuing" )
- ("jsonArray", "load a json array, not one item per line. Currently limited to 4MB." )
+ ("jsonArray", "load a json array, not one item per line. Currently limited to 16MB." )
;
add_hidden_options()
("noimport", "don't actually import. useful for benchmarking parser" )
Please sign in to comment.
Something went wrong with that request. Please try again.