Skip to content

Commit b24a851

Browse files
Fefer-IvanMongoDB Bot
Fefer-Ivan
authored and
MongoDB Bot
committed
SERVER-89078 Raise BufferMaxSize to 125 MB and allow passing BSONTraits to SBE fetchNext (#21044)
GitOrigin-RevId: 1c8ce898e491764a3d00fc879031fe2c54040ad6
1 parent e65b044 commit b24a851

File tree

10 files changed

+81
-37
lines changed

10 files changed

+81
-37
lines changed

‎jstests/aggregation/expressions/regex_limits.js

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -55,12 +55,12 @@ function testRegexAggException(inputObj, exceptionCode, expression) {
5555
// are 'n' characters in the input, it would result to 'n' individual matches. If the
5656
// pattern further has 'k' capture groups, then the output document will have 'n * k'
5757
// sub-strings representing the captures.
58-
const pattern = "(".repeat(150) + ")".repeat(150);
59-
// If the intermediate document size exceeds 64MB at any point, we will stop further
58+
const pattern = "(".repeat(150) + ")".repeat(150) + "(".repeat(150) + ")".repeat(150);
59+
// If the intermediate document size exceeds BufferMaxSize at any point, we will stop further
6060
// evaluation and throw an error.
6161
testRegexAggException({input: "$z", regex: pattern}, 51151, "$regexFindAll");
6262

63-
const pattern2 = "()".repeat(150);
63+
const pattern2 = "()".repeat(300);
6464
testRegexAggException({input: "$z", regex: pattern2}, 51151, "$regexFindAll");
6565
})();
6666

‎jstests/aggregation/sources/group/group_large_documents.js

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,6 @@
77
* # than maximum BSON size, we cannot serialize the result and send it to mongos. Such problem
88
* # does not exist in standalone and replica set setups.
99
* assumes_against_mongod_not_mongos,
10-
* # TODO SERVER-89078: Re-enable this test after we fix supporting huge intermediate docs
11-
* does_not_support_multiplanning_single_solutions,
1210
* ]
1311
*/
1412
import {arrayEq} from "jstests/aggregation/extras/utils.js";

‎jstests/aggregation/sources/lookup/lookup_large_documents.js

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,6 @@
44
* @tags: [
55
* # TODO SERVER-79448: Investigate why the test timeouts on TSAN variant.
66
* tsan_incompatible,
7-
* # TODO SERVER-89078: Re-enable this test after we fix supporting huge intermediate docs
8-
* does_not_support_multiplanning_single_solutions,
97
* ]
108
*/
119
import {arrayEq} from "jstests/aggregation/extras/utils.js";

‎src/mongo/bson/bson_obj_test.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -790,7 +790,7 @@ TEST(BSONObj, sizeChecks) {
790790
// But a size is in fact being enforced.
791791
ASSERT_THROWS_CODE(
792792
[&]() {
793-
auto hugeBuffer = generateBuffer(70 * 1024 * 1024);
793+
auto hugeBuffer = generateBuffer(130 * 1024 * 1024);
794794
BSONObj obj(hugeBuffer.data(), BSONObj::LargeSizeTrait{});
795795
}(),
796796
DBException,

‎src/mongo/bson/bsonobjbuilder_test.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -527,7 +527,7 @@ TEST(BSONObjBuilderTest, SizeChecks) {
527527

528528
// But a size is in fact being enforced.
529529
{
530-
auto largeBuffer = generateBuffer(40 * 1024 * 1024);
530+
auto largeBuffer = generateBuffer(50 * 1024 * 1024);
531531
BSONObj obj(largeBuffer.data(), BSONObj::LargeSizeTrait{});
532532
BSONObjBuilder builder;
533533
ASSERT_THROWS(

‎src/mongo/bson/util/builder.h

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -79,10 +79,16 @@ const int BSONObjMaxUserSize = 16 * 1024 * 1024;
7979
*/
8080
const int BSONObjMaxInternalSize = BSONObjMaxUserSize + (16 * 1024);
8181

82-
const int BufferMaxSize = 64 * 1024 * 1024;
82+
/**
83+
* Maximum size of a builder buffer and for BSONObj with BsonLargeSizeTrait. Limiting it to 27 bits
84+
* because SharedBuffer::Holder might bit pack information. Setting it to 125 MB to have some
85+
* wiggle room before size crosses 27 bits.
86+
*/
87+
const int BufferMaxSize = 125 * 1024 * 1024;
88+
static_assert(BufferMaxSize < (1 << 27));
8389

8490
/**
85-
* This is the maximum size size of a buffer needed for storing a BSON object in a response message.
91+
* This is the maximum size of a buffer needed for storing a BSON object in a response message.
8692
*/
8793
const int kOpMsgReplyBSONBufferMaxSize = BSONObjMaxUserSize + (64 * 1024);
8894

@@ -511,7 +517,8 @@ class BasicBufBuilder {
511517
// Going beyond the maximum buffer size is not likely.
512518
if (MONGO_unlikely(minSize > BufferMaxSize)) {
513519
std::stringstream ss;
514-
ss << "BufBuilder attempted to grow() to " << minSize << " bytes, past the 64MB limit.";
520+
ss << "BufBuilder attempted to grow() to " << minSize << " bytes, past the "
521+
<< (BufferMaxSize / (1024 * 1024)) << "MB limit.";
515522
msgasserted(13548, ss.str().c_str());
516523
}
517524

@@ -539,9 +546,9 @@ class BasicBufBuilder {
539546
} else if (MONGO_unlikely(reallocSize < 64)) {
540547
// The minimum allocation is 64 bytes.
541548
reallocSize = 64;
542-
} else if (MONGO_unlikely(minSize > BufferMaxSize)) {
543-
// If adding 'kBuffHolderSize' to 'minSize' pushes it beyond 'BufferMaxSize', then we'll
544-
// allocate enough memory according to the 'BufferMaxSize'.
549+
} else if (MONGO_unlikely(reallocSize + BufferAllocator::kBuffHolderSize > BufferMaxSize)) {
550+
// If adding 'kBuffHolderSize' to 'reallocSize' pushes it beyond 'BufferMaxSize', then
551+
// we'll allocate enough memory according to the 'BufferMaxSize'.
545552
reallocSize = BufferMaxSize + BufferAllocator::kBuffHolderSize;
546553
}
547554

‎src/mongo/db/query/plan_executor_sbe.cpp

Lines changed: 48 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -314,7 +314,7 @@ PlanExecutor::ExecState PlanExecutorSBE::getNext(BSONObj* out, RecordId* dlOut)
314314
return result;
315315
}
316316

317-
template <typename ObjectType>
317+
template <typename ObjectType, typename BSONTraits = BSONObj::DefaultSizeTrait>
318318
sbe::PlanState fetchNextImpl(sbe::PlanStage* root,
319319
sbe::value::SlotAccessor* resultSlot,
320320
sbe::value::SlotAccessor* recordIdSlot,
@@ -671,6 +671,7 @@ void PlanExecutorSBE::initializeAccessors(
671671
}
672672
}
673673

674+
template <typename BSONTraits>
674675
BSONObj PlanExecutorSBE::MetaDataAccessor::appendToBson(BSONObj doc) const {
675676
if (metadataSearchScore || metadataSearchHighlights || metadataSearchDetails ||
676677
metadataSearchSortValues || sortKey || metadataSearchSequenceToken) {
@@ -703,11 +704,16 @@ BSONObj PlanExecutorSBE::MetaDataAccessor::appendToBson(BSONObj doc) const {
703704
auto [tag, val] = metadataSearchSequenceToken->getViewOfValue();
704705
sbe::bson::appendValueToBsonObj(bb, Document::metaFieldSearchSequenceToken, tag, val);
705706
}
706-
return bb.obj();
707+
return bb.obj<BSONTraits>();
707708
}
708709
return doc;
709710
}
710711

712+
template BSONObj PlanExecutorSBE::MetaDataAccessor::appendToBson<BSONObj::DefaultSizeTrait>(
713+
BSONObj doc) const;
714+
template BSONObj PlanExecutorSBE::MetaDataAccessor::appendToBson<BSONObj::LargeSizeTrait>(
715+
BSONObj doc) const;
716+
711717
Document PlanExecutorSBE::MetaDataAccessor::appendToDocument(Document doc) const {
712718
if (metadataSearchScore || metadataSearchHighlights || metadataSearchDetails ||
713719
metadataSearchSortValues || sortKey || metadataSearchSequenceToken) {
@@ -772,7 +778,7 @@ Document PlanExecutorSBE::MetaDataAccessor::appendToDocument(Document doc) const
772778
return doc;
773779
}
774780

775-
template <typename ObjectType>
781+
template <typename ObjectType, typename BSONTraits>
776782
sbe::PlanState fetchNextImpl(sbe::PlanStage* root,
777783
sbe::value::SlotAccessor* resultSlot,
778784
sbe::value::SlotAccessor* recordIdSlot,
@@ -802,7 +808,7 @@ sbe::PlanState fetchNextImpl(sbe::PlanStage* root,
802808
if constexpr (isBson) {
803809
BSONObjBuilder bb;
804810
sbe::bson::convertToBsonObj(bb, sbe::value::getObjectView(val));
805-
*out = bb.obj();
811+
*out = bb.obj<BSONTraits>();
806812
} else {
807813
*out = convertToDocument(*sbe::value::getObjectView(val));
808814
}
@@ -818,7 +824,7 @@ sbe::PlanState fetchNextImpl(sbe::PlanStage* root,
818824
result = BSONObj{std::move(sharedBuf)};
819825
}
820826
} else {
821-
result = BSONObj{sbe::value::bitcastTo<const char*>(val)};
827+
result = BSONObj(sbe::value::bitcastTo<const char*>(val), BSONTraits{});
822828
}
823829

824830
if constexpr (isBson) {
@@ -834,7 +840,7 @@ sbe::PlanState fetchNextImpl(sbe::PlanStage* root,
834840
if constexpr (isDocument) {
835841
*out = metadata->appendToDocument(std::move(*out));
836842
} else {
837-
*out = metadata->appendToBson(std::move(*out));
843+
*out = metadata->appendToBson<BSONTraits>(std::move(*out));
838844
}
839845
}
840846
}
@@ -849,13 +855,23 @@ sbe::PlanState fetchNextImpl(sbe::PlanStage* root,
849855
return state;
850856
}
851857

852-
template sbe::PlanState fetchNextImpl<BSONObj>(sbe::PlanStage* root,
853-
sbe::value::SlotAccessor* resultSlot,
854-
sbe::value::SlotAccessor* recordIdSlot,
855-
BSONObj* out,
856-
RecordId* dlOut,
857-
bool returnOwnedBson,
858-
const PlanExecutorSBE::MetaDataAccessor* metadata);
858+
template sbe::PlanState fetchNextImpl<BSONObj, BSONObj::DefaultSizeTrait>(
859+
sbe::PlanStage* root,
860+
sbe::value::SlotAccessor* resultSlot,
861+
sbe::value::SlotAccessor* recordIdSlot,
862+
BSONObj* out,
863+
RecordId* dlOut,
864+
bool returnOwnedBson,
865+
const PlanExecutorSBE::MetaDataAccessor* metadata);
866+
867+
template sbe::PlanState fetchNextImpl<BSONObj, BSONObj::LargeSizeTrait>(
868+
sbe::PlanStage* root,
869+
sbe::value::SlotAccessor* resultSlot,
870+
sbe::value::SlotAccessor* recordIdSlot,
871+
BSONObj* out,
872+
RecordId* dlOut,
873+
bool returnOwnedBson,
874+
const PlanExecutorSBE::MetaDataAccessor* metadata);
859875

860876
template sbe::PlanState fetchNextImpl<Document>(sbe::PlanStage* root,
861877
sbe::value::SlotAccessor* resultSlot,
@@ -867,6 +883,7 @@ template sbe::PlanState fetchNextImpl<Document>(sbe::PlanStage* root,
867883

868884
// NOTE: We intentionally do not expose overload for the 'Document' type. The only interface to get
869885
// result from plan in 'Document' type is to call 'PlanExecutorSBE::getNextDocument()'.
886+
template <typename BSONTraits>
870887
sbe::PlanState fetchNext(sbe::PlanStage* root,
871888
sbe::value::SlotAccessor* resultSlot,
872889
sbe::value::SlotAccessor* recordIdSlot,
@@ -875,6 +892,23 @@ sbe::PlanState fetchNext(sbe::PlanStage* root,
875892
bool returnOwnedBson) {
876893
// Sending an empty MetaDataAccessor because we currently only deal with search related
877894
// metadata, and search query won't reach here.
878-
return fetchNextImpl(root, resultSlot, recordIdSlot, out, dlOut, returnOwnedBson, nullptr);
895+
return fetchNextImpl<BSONObj, BSONTraits>(
896+
root, resultSlot, recordIdSlot, out, dlOut, returnOwnedBson, nullptr);
879897
}
898+
899+
template sbe::PlanState fetchNext<BSONObj::DefaultSizeTrait>(sbe::PlanStage* root,
900+
sbe::value::SlotAccessor* resultSlot,
901+
sbe::value::SlotAccessor* recordIdSlot,
902+
BSONObj* out,
903+
RecordId* dlOut,
904+
bool returnOwnedBson);
905+
906+
template sbe::PlanState fetchNext<BSONObj::LargeSizeTrait>(sbe::PlanStage* root,
907+
sbe::value::SlotAccessor* resultSlot,
908+
sbe::value::SlotAccessor* recordIdSlot,
909+
BSONObj* out,
910+
RecordId* dlOut,
911+
bool returnOwnedBson);
912+
913+
880914
} // namespace mongo

‎src/mongo/db/query/plan_executor_sbe.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ namespace mongo {
7070
class PlanExecutorSBE final : public PlanExecutor {
7171
public:
7272
struct MetaDataAccessor {
73+
template <typename BSONTraits = BSONObj::DefaultSizeTrait>
7374
BSONObj appendToBson(BSONObj doc) const;
7475
Document appendToDocument(Document doc) const;
7576
// Only for $search queries, holds the metadata returned from mongot.
@@ -321,7 +322,11 @@ class PlanExecutorSBE final : public PlanExecutor {
321322
*
322323
* This common logic can be used by various consumers which need to fetch data using an SBE
323324
* PlanStage tree, such as PlanExecutor or RuntimePlanner.
325+
*
326+
* BSONTraits template parameter can be set to BSONObj::LargeSizeTrait if we want to allow resulting
327+
* BSONObj to be larged than 16 MB.
324328
*/
329+
template <typename BSONTraits = BSONObj::DefaultSizeTrait>
325330
sbe::PlanState fetchNext(sbe::PlanStage* root,
326331
sbe::value::SlotAccessor* resultSlot,
327332
sbe::value::SlotAccessor* recordIdSlot,

‎src/mongo/db/query/sbe_trial_runtime_executor.cpp

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -58,12 +58,14 @@ bool TrialRuntimeExecutor::fetchNextDocument(plan_ranker::CandidatePlan* candida
5858
BSONObj obj;
5959
RecordId recordId;
6060

61-
auto state = fetchNext(candidate->root.get(),
62-
resultSlot,
63-
recordIdSlot,
64-
&obj,
65-
recordIdSlot ? &recordId : nullptr,
66-
true /* must return owned BSON */);
61+
// SBE plan might be followed by an non-pushed down aggregation pipeline. In that case we
62+
// allow intermediate document to be large.
63+
auto state = fetchNext<BSONObj::LargeSizeTrait>(candidate->root.get(),
64+
resultSlot,
65+
recordIdSlot,
66+
&obj,
67+
recordIdSlot ? &recordId : nullptr,
68+
true /* must return owned BSON */);
6769
if (state == PlanState::IS_EOF) {
6870
candidate->root->close();
6971
return false;

‎src/mongo/dbtests/jsobjtests.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -219,8 +219,8 @@ class BufBuilderReallocLimit {
219219
BufBuilder b;
220220
unsigned int written = 0;
221221
try {
222-
for (; written <= 64 * 1024 * 1024 + 1; ++written)
223-
// (re)alloc past the buffer 64mb limit
222+
for (; written <= mongo::BufferMaxSize + 1; ++written)
223+
// (re)alloc past the buffer limit
224224
b.appendStr("a");
225225
} catch (const AssertionException&) {
226226
}

0 commit comments

Comments
 (0)