From e983feb7f977477d3da45697240f6d87d5cb890b Mon Sep 17 00:00:00 2001 From: Jeff Yemin Date: Wed, 27 Jan 2021 18:13:58 -0500 Subject: [PATCH 01/13] Fix or disable tests that fail against MongoDB 4.2 JAVA-3942 --- .evergreen/.evg.yml | 21 ++++++------------- .../client/CommandMonitoringTestHelper.java | 9 +++++--- .../AggregatesFunctionalSpecification.groovy | 8 ++----- .../AggregateOperationSpecification.groovy | 4 ++-- ...ateCollectionOperationSpecification.groovy | 6 +++--- ...CreateIndexesOperationSpecification.groovy | 2 +- .../FindOperationSpecification.groovy | 2 ++ ...ollectionScanOperationSpecification.groovy | 3 ++- .../com/mongodb/DBCollectionTest.java | 2 ++ .../functional/com/mongodb/DBCursorTest.java | 2 ++ .../test/functional/com/mongodb/DBTest.java | 6 +++++- 11 files changed, 33 insertions(+), 32 deletions(-) diff --git a/.evergreen/.evg.yml b/.evergreen/.evg.yml index bfa06c49db9..4683cb0e15c 100644 --- a/.evergreen/.evg.yml +++ b/.evergreen/.evg.yml @@ -316,15 +316,6 @@ functions: ${PREPARE_SHELL} echo '{"results": [{ "status": "FAIL", "test_file": "Build", "log_raw": "No test-results.json found was created" } ]}' > ${PROJECT_DIRECTORY}/test-results.json - "install dependencies": - type: test - params: - working_dir: "src" - script: | - ${PREPARE_SHELL} - file="${PROJECT_DIRECTORY}/.evergreen/install-dependencies.sh" - [ -f ${file} ] && sh ${file} || echo "${file} not available, skipping" - # Anchors hosts: &hosts @@ -337,7 +328,6 @@ pre: - func: "fix absolute paths" - func: "init test-results" - func: "make files executable" - - func: "install dependencies" post: # Removed, causing timeouts @@ -357,9 +347,6 @@ tasks: - func: "upload build" - name: "test" - depends_on: - - variant: "static-checks" - name: "static-analysis" commands: - func: "bootstrap mongo-orchestration" - func: "run tests" @@ -408,6 +395,10 @@ axes: - id: version display_name: MongoDB Version values: + - id: "4.2" + display_name: "4.2" + variables: + VERSION: "4.2" - id: "4.0" display_name: "4.0" variables: @@ -539,14 +530,14 @@ buildvariants: - matrix_name: "tests-jdk6-secure" matrix_spec: { auth: "auth", ssl: "ssl", jdk: "jdk6", version: "*", topology: "*", os: "*" } - exclude_spec: { auth: "auth", ssl: "ssl", jdk: "jdk6", version: ["4.0"], topology: "*", os: "*" } + exclude_spec: { auth: "auth", ssl: "ssl", jdk: "jdk6", version: ["4.0", "4.2"], topology: "*", os: "*" } display_name: "${version} ${topology} ${auth} ${ssl} ${jdk} ${os} " tags: ["tests-variant"] tasks: - name: "test" - matrix_name: "tests-jdk8-secure" - matrix_spec: { auth: "auth", ssl: "ssl", jdk: "jdk8", version: ["4.0"], topology: "*", os: "*" } + matrix_spec: { auth: "auth", ssl: "ssl", jdk: "jdk8", version: ["4.0", "4.2"], topology: "*", os: "*" } display_name: "${version} ${topology} ${auth} ${ssl} ${jdk} ${os} " tags: ["tests-variant"] tasks: diff --git a/driver-core/src/test/functional/com/mongodb/client/CommandMonitoringTestHelper.java b/driver-core/src/test/functional/com/mongodb/client/CommandMonitoringTestHelper.java index 79d96685733..15d5f0720c7 100644 --- a/driver-core/src/test/functional/com/mongodb/client/CommandMonitoringTestHelper.java +++ b/driver-core/src/test/functional/com/mongodb/client/CommandMonitoringTestHelper.java @@ -193,9 +193,12 @@ private static CommandSucceededEvent massageActualCommandSucceededEvent(final Co if (response.containsKey("writeErrors")) { for (BsonValue bsonValue : response.getArray("writeErrors")) { BsonDocument cur = bsonValue.asDocument(); - cur.put("code", new BsonInt32(42)); - cur.put("errmsg", new BsonString("")); - cur.remove("codeName"); + BsonDocument newWriteErrorDocument = + new BsonDocument().append("index", cur.get("index")) + .append("code", new BsonInt32(42)) + .append("errmsg", new BsonString("")); + cur.clear(); + cur.putAll(newWriteErrorDocument); } } if (actual.getCommandName().equals("update")) { diff --git a/driver-core/src/test/functional/com/mongodb/client/model/AggregatesFunctionalSpecification.groovy b/driver-core/src/test/functional/com/mongodb/client/model/AggregatesFunctionalSpecification.groovy index 7a25d0d3e9a..13d05aa8468 100644 --- a/driver-core/src/test/functional/com/mongodb/client/model/AggregatesFunctionalSpecification.groovy +++ b/driver-core/src/test/functional/com/mongodb/client/model/AggregatesFunctionalSpecification.groovy @@ -674,7 +674,6 @@ class AggregatesFunctionalSpecification extends OperationFunctionalSpecification helper.drop() helper.insertDocuments(Document.parse('{_id: 0, x: 1}')) - helper.insertDocuments(Document.parse('{_id: 1, x: 2}')) helper.insertDocuments(Document.parse('{_id: 2, x: 1}')) helper.insertDocuments(Document.parse('{_id: 3, x: 0}')) @@ -682,14 +681,12 @@ class AggregatesFunctionalSpecification extends OperationFunctionalSpecification then: results == [Document.parse('{_id: 1, count: 2}'), - Document.parse('{_id: 0, count: 1}'), - Document.parse('{_id: 2, count: 1}')] + Document.parse('{_id: 0, count: 1}')] when: helper.drop() helper.insertDocuments(Document.parse('{_id: 0, x: 1.4}')) - helper.insertDocuments(Document.parse('{_id: 1, x: 2.3}')) helper.insertDocuments(Document.parse('{_id: 2, x: 1.1}')) helper.insertDocuments(Document.parse('{_id: 3, x: 0.5}')) @@ -697,8 +694,7 @@ class AggregatesFunctionalSpecification extends OperationFunctionalSpecification then: results == [Document.parse('{_id: 1, count: 2}'), - Document.parse('{_id: 0, count: 1}'), - Document.parse('{_id: 2, count: 1}')] + Document.parse('{_id: 0, count: 1}')] cleanup: helper?.drop() diff --git a/driver-core/src/test/functional/com/mongodb/operation/AggregateOperationSpecification.groovy b/driver-core/src/test/functional/com/mongodb/operation/AggregateOperationSpecification.groovy index 54d7cd4ed1a..8f14da6436d 100644 --- a/driver-core/src/test/functional/com/mongodb/operation/AggregateOperationSpecification.groovy +++ b/driver-core/src/test/functional/com/mongodb/operation/AggregateOperationSpecification.groovy @@ -354,7 +354,7 @@ class AggregateOperationSpecification extends OperationFunctionalSpecification { def result = execute(operation, async) then: - result.containsKey('stages') + result.containsKey('stages') || result.containsKey('queryPlanner') where: async << [true, false] @@ -377,7 +377,7 @@ class AggregateOperationSpecification extends OperationFunctionalSpecification { [async, options] << [[true, false], [defaultCollation, null, Collation.builder().build()]].combinations() } - @IgnoreIf({ !serverVersionAtLeast(3, 6) }) + @IgnoreIf({ !serverVersionAtLeast(3, 6) || serverVersionAtLeast(4, 2) }) def 'should apply $hint'() { given: def index = new BsonDocument('a', new BsonInt32(1)) diff --git a/driver-core/src/test/functional/com/mongodb/operation/CreateCollectionOperationSpecification.groovy b/driver-core/src/test/functional/com/mongodb/operation/CreateCollectionOperationSpecification.groovy index 1b2c598047c..fc770f515cb 100644 --- a/driver-core/src/test/functional/com/mongodb/operation/CreateCollectionOperationSpecification.groovy +++ b/driver-core/src/test/functional/com/mongodb/operation/CreateCollectionOperationSpecification.groovy @@ -108,8 +108,7 @@ class CreateCollectionOperationSpecification extends OperationFunctionalSpecific given: def operation = new CreateCollectionOperation(getDatabaseName(), getCollectionName()) .storageEngineOptions(new BsonDocument('wiredTiger', - new BsonDocument('configString', new BsonString('block_compressor=zlib'))) - .append('mmapv1', new BsonDocument())) + new BsonDocument('configString', new BsonString('block_compressor=zlib')))) when: execute(operation, async) @@ -123,6 +122,7 @@ class CreateCollectionOperationSpecification extends OperationFunctionalSpecific async << [true, false] } + @IgnoreIf( {serverVersionAtLeast(4, 2) }) def 'should set flags for use power of two sizes'() { given: assert !collectionNameExists(getCollectionName()) @@ -195,7 +195,7 @@ class CreateCollectionOperationSpecification extends OperationFunctionalSpecific false | 0 | false } - @IgnoreIf({ !serverVersionAtLeast(3, 2) }) + @IgnoreIf({ !serverVersionAtLeast(3, 2) || serverVersionAtLeast(4, 2)}) def 'should allow indexOptionDefaults'() { given: assert !collectionNameExists(getCollectionName()) diff --git a/driver-core/src/test/functional/com/mongodb/operation/CreateIndexesOperationSpecification.groovy b/driver-core/src/test/functional/com/mongodb/operation/CreateIndexesOperationSpecification.groovy index 925b6f7adf5..cfbab9cb7fe 100644 --- a/driver-core/src/test/functional/com/mongodb/operation/CreateIndexesOperationSpecification.groovy +++ b/driver-core/src/test/functional/com/mongodb/operation/CreateIndexesOperationSpecification.groovy @@ -389,7 +389,7 @@ class CreateIndexesOperationSpecification extends OperationFunctionalSpecificati async << [true, false] } - @IgnoreIf({ !serverVersionAtLeast(3, 0) }) + @IgnoreIf({ !serverVersionAtLeast(3, 0) || serverVersionAtLeast(4, 2)}) def 'should pass through storage engine options'() { given: def storageEngineOptions = new Document('wiredTiger', new Document('configString', 'block_compressor=zlib')) diff --git a/driver-core/src/test/functional/com/mongodb/operation/FindOperationSpecification.groovy b/driver-core/src/test/functional/com/mongodb/operation/FindOperationSpecification.groovy index 49dfdfc5b59..259fbdd9597 100644 --- a/driver-core/src/test/functional/com/mongodb/operation/FindOperationSpecification.groovy +++ b/driver-core/src/test/functional/com/mongodb/operation/FindOperationSpecification.groovy @@ -351,6 +351,7 @@ class FindOperationSpecification extends OperationFunctionalSpecification { ].combinations() } + @IgnoreIf( {serverVersionAtLeast(4, 2) }) def '$max should limit items returned'() { given: (1..100).each { @@ -370,6 +371,7 @@ class FindOperationSpecification extends OperationFunctionalSpecification { async << [true, false] } + @IgnoreIf( {serverVersionAtLeast(4, 2) }) def '$min should limit items returned'() { given: (1..100).each { diff --git a/driver-core/src/test/functional/com/mongodb/operation/ParallelCollectionScanOperationSpecification.groovy b/driver-core/src/test/functional/com/mongodb/operation/ParallelCollectionScanOperationSpecification.groovy index e09f7181cda..4bac00fc795 100644 --- a/driver-core/src/test/functional/com/mongodb/operation/ParallelCollectionScanOperationSpecification.groovy +++ b/driver-core/src/test/functional/com/mongodb/operation/ParallelCollectionScanOperationSpecification.groovy @@ -49,9 +49,10 @@ import static com.mongodb.ClusterFixture.executeAsync import static com.mongodb.ClusterFixture.getBinding import static com.mongodb.ClusterFixture.isSharded import static com.mongodb.ClusterFixture.loopCursor +import static com.mongodb.ClusterFixture.serverVersionAtLeast import static org.junit.Assert.assertTrue -@IgnoreIf({ isSharded() }) +@IgnoreIf({ isSharded() || serverVersionAtLeast(4, 2) } ) @Category(Slow) class ParallelCollectionScanOperationSpecification extends OperationFunctionalSpecification { Map ids = [] as ConcurrentHashMap diff --git a/driver-legacy/src/test/functional/com/mongodb/DBCollectionTest.java b/driver-legacy/src/test/functional/com/mongodb/DBCollectionTest.java index 6183a0b6e48..27b8f1f8e48 100644 --- a/driver-legacy/src/test/functional/com/mongodb/DBCollectionTest.java +++ b/driver-legacy/src/test/functional/com/mongodb/DBCollectionTest.java @@ -52,6 +52,7 @@ import static com.mongodb.ClusterFixture.isDiscoverableReplicaSet; import static com.mongodb.ClusterFixture.isSharded; import static com.mongodb.ClusterFixture.serverVersionAtLeast; +import static com.mongodb.ClusterFixture.serverVersionLessThan; import static com.mongodb.DBObjectMatchers.hasSubdocument; import static java.util.Arrays.asList; import static java.util.concurrent.TimeUnit.MILLISECONDS; @@ -969,6 +970,7 @@ public void testBulkWriteConcernException() throws UnknownHostException { @Category(Slow.class) public void testParallelScan() throws UnknownHostException { assumeThat(isSharded(), is(false)); + assumeThat(serverVersionLessThan("4.2"), is(true)); Set ids = new HashSet(); List documents = new ArrayList(2000); diff --git a/driver-legacy/src/test/functional/com/mongodb/DBCursorTest.java b/driver-legacy/src/test/functional/com/mongodb/DBCursorTest.java index 8ad73a60812..3efd4ae4642 100644 --- a/driver-legacy/src/test/functional/com/mongodb/DBCursorTest.java +++ b/driver-legacy/src/test/functional/com/mongodb/DBCursorTest.java @@ -320,6 +320,7 @@ public void testMaxScan() { @Test public void testMax() { + assumeFalse(serverVersionAtLeast(4, 2)); countResults(new DBCursor(collection, new BasicDBObject(), new BasicDBObject(), ReadPreference.primary()) .addSpecial("$max", new BasicDBObject("x", 4)), 4); countResults(new DBCursor(collection, new BasicDBObject(), new BasicDBObject(), ReadPreference.primary()) @@ -328,6 +329,7 @@ public void testMax() { @Test public void testMin() { + assumeFalse(serverVersionAtLeast(4, 2)); countResults(new DBCursor(collection, new BasicDBObject(), new BasicDBObject(), ReadPreference.primary()) .addSpecial("$min", new BasicDBObject("x", 4)), 6); countResults(new DBCursor(collection, new BasicDBObject(), new BasicDBObject(), ReadPreference.primary()) diff --git a/driver-legacy/src/test/functional/com/mongodb/DBTest.java b/driver-legacy/src/test/functional/com/mongodb/DBTest.java index fdbac06ba37..c4154129da2 100644 --- a/driver-legacy/src/test/functional/com/mongodb/DBTest.java +++ b/driver-legacy/src/test/functional/com/mongodb/DBTest.java @@ -38,6 +38,7 @@ import static com.mongodb.ClusterFixture.isDiscoverableReplicaSet; import static com.mongodb.ClusterFixture.isSharded; import static com.mongodb.ClusterFixture.serverVersionAtLeast; +import static com.mongodb.ClusterFixture.serverVersionLessThan; import static com.mongodb.DBObjectMatchers.hasFields; import static com.mongodb.DBObjectMatchers.hasSubdocument; import static com.mongodb.Fixture.getDefaultDatabaseName; @@ -218,6 +219,7 @@ public void shouldGetDuplicateKeyException() { @Test public void shouldDoEval() { assumeThat(isAuthenticated(), is(false)); + assumeThat(serverVersionLessThan("4.2"), is(true)); String code = "function(name, incAmount) {\n" + "var doc = db.myCollection.findOne( { name : name } );\n" + "doc = doc || { name : name , num : 0 , total : 0 , avg : 0 , _id: 1 };\n" @@ -237,6 +239,7 @@ public void shouldDoEval() { @Test(expected = MongoException.class) public void shouldThrowErrorwhileDoingEval() { + assumeThat(serverVersionLessThan("4.2"), is(true)); String code = "function(a, b) {\n" + "var doc = db.myCollection.findOne( { name : b } );\n" + "}"; @@ -246,6 +249,7 @@ public void shouldThrowErrorwhileDoingEval() { @Test public void shouldInsertDocumentsUsingEval() { assumeThat(isAuthenticated(), is(false)); + assumeThat(serverVersionLessThan("4.2"), is(true)); // when database.eval("db." + collectionName + ".insert({name: 'Bob'})"); @@ -286,7 +290,7 @@ public void shouldExecuteCommandWithReadPreference() { @Test public void shouldNotThrowAnExceptionOnCommandFailure() { - CommandResult commandResult = database.command(new BasicDBObject("collStats", "a" + System.currentTimeMillis())); + CommandResult commandResult = database.command(new BasicDBObject("nonExistentCommand", 1)); assertThat(commandResult, hasFields(new String[]{"ok", "errmsg"})); } From ad624f43e31d1b82c72bca4384567c3677f5e8a9 Mon Sep 17 00:00:00 2001 From: Jeff Yemin Date: Thu, 28 Jan 2021 13:55:33 -0500 Subject: [PATCH 02/13] Disable test for change stream projecting out the _id field There's an integration test that asserts an exception is thrown when a change stream projects out the _id field. Running against 3.6 and 4.0, this condition is checked in the driver itself while processing the results. But in 4.4 the server started replying with an error on the getMore in this scenario. A newer driver would know not to retry in this case, because newer drivers implemented an allow list for resumeable change stream errors when the server version. But older drivers don't have that logic, so it will treat this command error as retryable and never actually report the error. JAVA-3942 --- .../mongodb/operation/ChangeStreamOperationSpecification.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/driver-core/src/test/functional/com/mongodb/operation/ChangeStreamOperationSpecification.groovy b/driver-core/src/test/functional/com/mongodb/operation/ChangeStreamOperationSpecification.groovy index e95c8894948..f315e15c954 100644 --- a/driver-core/src/test/functional/com/mongodb/operation/ChangeStreamOperationSpecification.groovy +++ b/driver-core/src/test/functional/com/mongodb/operation/ChangeStreamOperationSpecification.groovy @@ -390,6 +390,7 @@ class ChangeStreamOperationSpecification extends OperationFunctionalSpecificatio waitForLastRelease(getCluster()) } + @IgnoreIf({ serverVersionAtLeast([4, 2, 0]) }) def 'should throw if the _id field is projected out'() { given: def helper = getHelper() From ed6ea84a471ebcaf693518502cb16b5438120b2e Mon Sep 17 00:00:00 2001 From: Jeff Yemin Date: Thu, 28 Jan 2021 13:56:37 -0500 Subject: [PATCH 03/13] Make change stream operation integration test more robust JAVA-3942 --- .../ChangeStreamOperationSpecification.groovy | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/driver-core/src/test/functional/com/mongodb/operation/ChangeStreamOperationSpecification.groovy b/driver-core/src/test/functional/com/mongodb/operation/ChangeStreamOperationSpecification.groovy index f315e15c954..2b5aa33c78b 100644 --- a/driver-core/src/test/functional/com/mongodb/operation/ChangeStreamOperationSpecification.groovy +++ b/driver-core/src/test/functional/com/mongodb/operation/ChangeStreamOperationSpecification.groovy @@ -466,9 +466,13 @@ class ChangeStreamOperationSpecification extends OperationFunctionalSpecificatio when: helper.killCursor(helper.getNamespace(), cursor.getWrapped().getServerCursor()) expected = insertDocuments(helper, [3, 4]) + def results = nextAndClean(cursor, async) + if (results.size() < expected.size()) { + results.addAll(nextAndClean(cursor, async)) + } then: - nextAndClean(cursor, async) == expected + results == expected then: tryNextAndClean(cursor, async) == null @@ -476,9 +480,13 @@ class ChangeStreamOperationSpecification extends OperationFunctionalSpecificatio when: expected = insertDocuments(helper, [5, 6]) helper.killCursor(helper.getNamespace(), cursor.getWrapped().getServerCursor()) - + results = nextAndClean(cursor, async) + if (results.size() < expected.size()) { + results.addAll(nextAndClean(cursor, async)) + } + then: - nextAndClean(cursor, async) == expected + results == expected cleanup: cursor?.close() From a5a63f69fce96a44a7b9c22dcb83f493ff899782 Mon Sep 17 00:00:00 2001 From: Jeff Yemin Date: Thu, 28 Jan 2021 13:29:31 -0500 Subject: [PATCH 04/13] Disable $hint test for 4.2+ sharded clusters The reply format has changed in a way that breaks the test. This test has been disabled in subsequent releases and ultimately just removed. JAVA-3942 --- .../AggregateToCollectionOperationSpecification.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/driver-core/src/test/functional/com/mongodb/operation/AggregateToCollectionOperationSpecification.groovy b/driver-core/src/test/functional/com/mongodb/operation/AggregateToCollectionOperationSpecification.groovy index 6d872956d12..97d95c5c2ee 100644 --- a/driver-core/src/test/functional/com/mongodb/operation/AggregateToCollectionOperationSpecification.groovy +++ b/driver-core/src/test/functional/com/mongodb/operation/AggregateToCollectionOperationSpecification.groovy @@ -291,7 +291,7 @@ class AggregateToCollectionOperationSpecification extends OperationFunctionalSpe async << [true, false] } - @IgnoreIf({ !serverVersionAtLeast(3, 6) }) + @IgnoreIf({ !serverVersionAtLeast(3, 6) || (serverVersionAtLeast(4, 2) && isSharded()) }) def 'should apply $hint'() { given: def hint = new BsonDocument('a', new BsonInt32(1)) From 129d7db0f33bfd2e6ec20c608d4ec5f181ea88cc Mon Sep 17 00:00:00 2001 From: Jeff Yemin Date: Wed, 27 Jan 2021 19:56:19 -0500 Subject: [PATCH 05/13] Make mapReduce implementation compatible with 4.4 JAVA-3942 --- .../operation/ServerVersionHelper.java | 4 +++ .../mongodb/operation/MapReduceHelper.java | 9 ++++--- .../MapReduceToCollectionOperation.java | 26 +++++++++++-------- .../MapReduceWithInlineResultsOperation.java | 18 +++++++------ 4 files changed, 34 insertions(+), 23 deletions(-) diff --git a/driver-core/src/main/com/mongodb/internal/operation/ServerVersionHelper.java b/driver-core/src/main/com/mongodb/internal/operation/ServerVersionHelper.java index c16101007bf..627340850ca 100644 --- a/driver-core/src/main/com/mongodb/internal/operation/ServerVersionHelper.java +++ b/driver-core/src/main/com/mongodb/internal/operation/ServerVersionHelper.java @@ -45,6 +45,10 @@ public static boolean serverIsAtLeastVersionFourDotZero(final ConnectionDescript return serverIsAtLeastVersion(description, new ServerVersion(4, 0)); } + public static boolean serverIsAtLeastVersionFourDotFour(final ConnectionDescription description) { + return serverIsAtLeastVersion(description, new ServerVersion(4, 4)); + } + private static boolean serverIsAtLeastVersion(final ConnectionDescription description, final ServerVersion serverVersion) { return description.getServerVersion().compareTo(serverVersion) >= 0; } diff --git a/driver-core/src/main/com/mongodb/operation/MapReduceHelper.java b/driver-core/src/main/com/mongodb/operation/MapReduceHelper.java index bfaf8c217e0..bfd054fc2b2 100644 --- a/driver-core/src/main/com/mongodb/operation/MapReduceHelper.java +++ b/driver-core/src/main/com/mongodb/operation/MapReduceHelper.java @@ -17,6 +17,7 @@ package com.mongodb.operation; import org.bson.BsonDocument; +import org.bson.BsonInt32; final class MapReduceHelper { @@ -26,19 +27,19 @@ static MapReduceStatistics createStatistics(final BsonDocument result) { } private static int getInputCount(final BsonDocument result) { - return result.getDocument("counts").getNumber("input").intValue(); + return result.getDocument("counts", new BsonDocument()).getNumber("input", new BsonInt32(0)).intValue(); } private static int getOutputCount(final BsonDocument result) { - return result.getDocument("counts").getNumber("output").intValue(); + return result.getDocument("counts", new BsonDocument()).getNumber("output", new BsonInt32(0)).intValue(); } private static int getEmitCount(final BsonDocument result) { - return result.getDocument("counts").getNumber("emit").intValue(); + return result.getDocument("counts", new BsonDocument()).getNumber("emit", new BsonInt32(0)).intValue(); } private static int getDuration(final BsonDocument result) { - return result.getNumber("timeMillis").intValue(); + return result.getNumber("timeMillis", new BsonInt32(0)).intValue(); } private MapReduceHelper() { diff --git a/driver-core/src/main/com/mongodb/operation/MapReduceToCollectionOperation.java b/driver-core/src/main/com/mongodb/operation/MapReduceToCollectionOperation.java index 2ffa7f9f73d..67b796378ec 100644 --- a/driver-core/src/main/com/mongodb/operation/MapReduceToCollectionOperation.java +++ b/driver-core/src/main/com/mongodb/operation/MapReduceToCollectionOperation.java @@ -42,8 +42,10 @@ import static com.mongodb.assertions.Assertions.isTrue; import static com.mongodb.assertions.Assertions.notNull; import static com.mongodb.internal.async.ErrorHandlingResultCallback.errorHandlingCallback; +import static com.mongodb.internal.operation.ServerVersionHelper.serverIsAtLeastVersionFourDotFour; import static com.mongodb.operation.CommandOperationHelper.executeWrappedCommandProtocol; import static com.mongodb.operation.CommandOperationHelper.executeWrappedCommandProtocolAsync; +import static com.mongodb.operation.DocumentHelper.putIfNotNull; import static com.mongodb.operation.DocumentHelper.putIfNotZero; import static com.mongodb.operation.DocumentHelper.putIfTrue; import static com.mongodb.operation.OperationHelper.AsyncCallableWithConnection; @@ -581,21 +583,23 @@ public MapReduceStatistics apply(final BsonDocument result, final ServerAddress private BsonDocument getCommand(final ConnectionDescription description) { BsonDocument outputDocument = new BsonDocument(getAction(), new BsonString(getCollectionName())); - outputDocument.append("sharded", BsonBoolean.valueOf(isSharded())); - outputDocument.append("nonAtomic", BsonBoolean.valueOf(isNonAtomic())); + if (description != null && !serverIsAtLeastVersionFourDotFour(description)) { + putIfTrue(outputDocument, "sharded", isSharded()); + putIfTrue(outputDocument, "nonAtomic", isNonAtomic()); + } if (getDatabaseName() != null) { outputDocument.put("db", new BsonString(getDatabaseName())); } BsonDocument commandDocument = new BsonDocument("mapreduce", new BsonString(namespace.getCollectionName())) - .append("map", getMapFunction()) - .append("reduce", getReduceFunction()) - .append("out", outputDocument) - .append("query", asValueOrNull(getFilter())) - .append("sort", asValueOrNull(getSort())) - .append("finalize", asValueOrNull(getFinalizeFunction())) - .append("scope", asValueOrNull(getScope())) - .append("verbose", BsonBoolean.valueOf(isVerbose())); - putIfNotZero(commandDocument, "limit", getLimit()); + .append("map", getMapFunction()) + .append("reduce", getReduceFunction()) + .append("out", outputDocument); + + putIfNotNull(commandDocument, "query", getFilter()); + putIfNotNull(commandDocument, "sort", getSort()); + putIfNotNull(commandDocument, "finalize", getFinalizeFunction()); + putIfNotNull(commandDocument, "scope", getScope()); + putIfTrue(commandDocument, "verbose", isVerbose()); putIfNotZero(commandDocument, "limit", getLimit()); putIfNotZero(commandDocument, "maxTimeMS", getMaxTime(MILLISECONDS)); putIfTrue(commandDocument, "jsMode", isJsMode()); if (bypassDocumentValidation != null && description != null && serverIsAtLeastVersionThreeDotTwo(description)) { diff --git a/driver-core/src/main/com/mongodb/operation/MapReduceWithInlineResultsOperation.java b/driver-core/src/main/com/mongodb/operation/MapReduceWithInlineResultsOperation.java index 02ebce53fbd..b3fad2612c4 100644 --- a/driver-core/src/main/com/mongodb/operation/MapReduceWithInlineResultsOperation.java +++ b/driver-core/src/main/com/mongodb/operation/MapReduceWithInlineResultsOperation.java @@ -48,6 +48,7 @@ import static com.mongodb.internal.async.ErrorHandlingResultCallback.errorHandlingCallback; import static com.mongodb.operation.CommandOperationHelper.executeWrappedCommandProtocol; import static com.mongodb.operation.CommandOperationHelper.executeWrappedCommandProtocolAsync; +import static com.mongodb.operation.DocumentHelper.putIfNotNull; import static com.mongodb.operation.DocumentHelper.putIfNotZero; import static com.mongodb.operation.DocumentHelper.putIfTrue; import static com.mongodb.operation.ExplainHelper.asExplainCommand; @@ -447,14 +448,15 @@ public MapReduceAsyncBatchCursor apply(final BsonDocument result, final Serve private BsonDocument getCommand(final SessionContext sessionContext) { BsonDocument commandDocument = new BsonDocument("mapreduce", new BsonString(namespace.getCollectionName())) - .append("map", getMapFunction()) - .append("reduce", getReduceFunction()) - .append("out", new BsonDocument("inline", new BsonInt32(1))) - .append("query", asValueOrNull(getFilter())) - .append("sort", asValueOrNull(getSort())) - .append("finalize", asValueOrNull(getFinalizeFunction())) - .append("scope", asValueOrNull(getScope())) - .append("verbose", BsonBoolean.valueOf(isVerbose())); + .append("map", getMapFunction()) + .append("reduce", getReduceFunction()) + .append("out", new BsonDocument("inline", new BsonInt32(1))); + + putIfNotNull(commandDocument, "query", getFilter()); + putIfNotNull(commandDocument, "sort", getSort()); + putIfNotNull(commandDocument, "finalize", getFinalizeFunction()); + putIfNotNull(commandDocument, "scope", getScope()); + putIfTrue(commandDocument, "verbose", isVerbose()); appendReadConcernToCommand(sessionContext, commandDocument); putIfNotZero(commandDocument, "limit", getLimit()); From ab13eab8434a11465b08e39f5238c36d6690fd6d Mon Sep 17 00:00:00 2001 From: Jeff Yemin Date: Wed, 27 Jan 2021 19:57:06 -0500 Subject: [PATCH 06/13] Make mapReduce tests compatible with 4.4 JAVA-3942 --- ...eToCollectionOperationSpecification.groovy | 37 +++++++--- ...InlineResultsOperationSpecification.groovy | 74 ++++++++----------- .../functional/com/mongodb/MapReduceTest.java | 29 ++++++-- .../mongodb/client/MongoCollectionTest.java | 5 +- 4 files changed, 81 insertions(+), 64 deletions(-) diff --git a/driver-core/src/test/functional/com/mongodb/operation/MapReduceToCollectionOperationSpecification.groovy b/driver-core/src/test/functional/com/mongodb/operation/MapReduceToCollectionOperationSpecification.groovy index d0aa76f8b56..a494e6edc89 100644 --- a/driver-core/src/test/functional/com/mongodb/operation/MapReduceToCollectionOperationSpecification.groovy +++ b/driver-core/src/test/functional/com/mongodb/operation/MapReduceToCollectionOperationSpecification.groovy @@ -26,18 +26,22 @@ import com.mongodb.client.model.ValidationOptions import com.mongodb.client.test.CollectionHelper import org.bson.BsonBoolean import org.bson.BsonDocument +import org.bson.BsonDouble import org.bson.BsonInt32 import org.bson.BsonInt64 import org.bson.BsonJavaScript import org.bson.BsonNull import org.bson.BsonString import org.bson.Document +import org.bson.codecs.BsonDocumentCodec import org.bson.codecs.DocumentCodec import spock.lang.IgnoreIf import static com.mongodb.ClusterFixture.getBinding import static com.mongodb.ClusterFixture.isDiscoverableReplicaSet import static com.mongodb.ClusterFixture.serverVersionAtLeast +import static com.mongodb.ClusterFixture.serverVersionGreaterThan +import static com.mongodb.ClusterFixture.serverVersionLessThan import static com.mongodb.client.model.Filters.gte import static java.util.concurrent.TimeUnit.MILLISECONDS @@ -48,10 +52,10 @@ class MapReduceToCollectionOperationSpecification extends OperationFunctionalSpe new BsonJavaScript('function(){ emit( this.name , 1 ); }'), new BsonJavaScript('function(key, values){ return values.length; }'), mapReduceOutputNamespace.getCollectionName()) - def expectedResults = [['_id': 'Pete', 'value': 2.0] as Document, - ['_id': 'Sam', 'value': 1.0] as Document] - def helper = new CollectionHelper(new DocumentCodec(), mapReduceOutputNamespace) - + def expectedResults = [new BsonDocument('_id', new BsonString('Pete')).append('value', new BsonDouble(2.0)), + new BsonDocument('_id', new BsonString('Sam')).append('value', new BsonDouble(1.0))] as Set + def helper = new CollectionHelper(new BsonDocumentCodec(), mapReduceOutputNamespace) + def setup() { CollectionHelper helper = new CollectionHelper(new DocumentCodec(), mapReduceInputNamespace) Document pete = new Document('name', 'Pete').append('job', 'handyman') @@ -137,6 +141,7 @@ class MapReduceToCollectionOperationSpecification extends OperationFunctionalSpe operation.getCollation() == defaultCollation } + @IgnoreIf({ serverVersionGreaterThan('4.2') }) def 'should return the correct statistics and save the results'() { when: MapReduceStatistics results = execute(mapReduceOperation, async) @@ -152,6 +157,21 @@ class MapReduceToCollectionOperationSpecification extends OperationFunctionalSpe async << [true, false] } + @IgnoreIf({ serverVersionLessThan('4.4') }) + def 'should return zero-valued statistics and save the results'() { + when: + MapReduceStatistics results = execute(mapReduceOperation, async) + + then: + results.emitCount == 0 + results.inputCount == 0 + results.outputCount == 0 + helper.count() == 2 + helper.find() as Set == expectedResults + + where: + async << [true, false] + } @IgnoreIf({ !serverVersionAtLeast(3, 2) }) def 'should support bypassDocumentValidation'() { @@ -232,12 +252,7 @@ class MapReduceToCollectionOperationSpecification extends OperationFunctionalSpe def expectedCommand = new BsonDocument('mapreduce', new BsonString(getCollectionName())) .append('map', mapF) .append('reduce', reduceF) - .append('out', BsonDocument.parse('{replace: "outCollection", sharded: false, nonAtomic: false}')) - .append('query', BsonNull.VALUE) - .append('sort', BsonNull.VALUE) - .append('finalize', BsonNull.VALUE) - .append('scope', BsonNull.VALUE) - .append('verbose', BsonBoolean.FALSE) + .append('out', BsonDocument.parse('{replace: "outCollection"}')) if (includeWriteConcern) { expectedCommand.append('writeConcern', WriteConcern.MAJORITY.asDocument()) @@ -258,7 +273,7 @@ class MapReduceToCollectionOperationSpecification extends OperationFunctionalSpe .bypassDocumentValidation(true) .verbose(true) - expectedCommand.append('out', BsonDocument.parse('{merge: "outCollection", sharded: false, nonAtomic: false, db: "dbName"}')) + expectedCommand.append('out', BsonDocument.parse('{merge: "outCollection", db: "dbName"}')) .append('query', filter) .append('sort', sort) .append('finalize', finalizeF) diff --git a/driver-core/src/test/functional/com/mongodb/operation/MapReduceWithInlineResultsOperationSpecification.groovy b/driver-core/src/test/functional/com/mongodb/operation/MapReduceWithInlineResultsOperationSpecification.groovy index 54c229c496b..497c6ec594e 100644 --- a/driver-core/src/test/functional/com/mongodb/operation/MapReduceWithInlineResultsOperationSpecification.groovy +++ b/driver-core/src/test/functional/com/mongodb/operation/MapReduceWithInlineResultsOperationSpecification.groovy @@ -36,13 +36,14 @@ import com.mongodb.connection.ServerVersion import com.mongodb.session.SessionContext import org.bson.BsonBoolean import org.bson.BsonDocument +import org.bson.BsonDouble import org.bson.BsonInt32 import org.bson.BsonInt64 import org.bson.BsonJavaScript -import org.bson.BsonNull import org.bson.BsonString import org.bson.BsonTimestamp import org.bson.Document +import org.bson.codecs.BsonDocumentCodec import org.bson.codecs.DocumentCodec import spock.lang.IgnoreIf @@ -53,18 +54,18 @@ import static com.mongodb.operation.OperationReadConcernHelper.appendReadConcern import static java.util.concurrent.TimeUnit.MILLISECONDS class MapReduceWithInlineResultsOperationSpecification extends OperationFunctionalSpecification { - private final documentCodec = new DocumentCodec() - def mapReduceOperation = new MapReduceWithInlineResultsOperation( + private final bsonDocumentCodec = new BsonDocumentCodec() + def mapReduceOperation = new MapReduceWithInlineResultsOperation( getNamespace(), new BsonJavaScript('function(){ emit( this.name , 1 ); }'), new BsonJavaScript('function(key, values){ return values.length; }'), - documentCodec) + bsonDocumentCodec) - def expectedResults = [['_id': 'Pete', 'value': 2.0] as Document, - ['_id': 'Sam', 'value': 1.0] as Document] + def expectedResults = [new BsonDocument('_id', new BsonString('Pete')).append('value', new BsonDouble(2.0)), + new BsonDocument('_id', new BsonString('Sam')).append('value', new BsonDouble(1.0))] as Set def setup() { - CollectionHelper helper = new CollectionHelper(documentCodec, getNamespace()) + CollectionHelper helper = new CollectionHelper(bsonDocumentCodec, getNamespace()) Document pete = new Document('name', 'Pete').append('job', 'handyman') Document sam = new Document('name', 'Sam').append('job', 'plumber') Document pete2 = new Document('name', 'Pete').append('job', 'electrician') @@ -75,7 +76,7 @@ class MapReduceWithInlineResultsOperationSpecification extends OperationFunction when: def mapF = new BsonJavaScript('function(){ }') def reduceF = new BsonJavaScript('function(key, values){ }') - def operation = new MapReduceWithInlineResultsOperation(helper.namespace, mapF, reduceF, documentCodec) + def operation = new MapReduceWithInlineResultsOperation(helper.namespace, mapF, reduceF, bsonDocumentCodec) then: operation.getMapFunction() == mapF @@ -99,7 +100,7 @@ class MapReduceWithInlineResultsOperationSpecification extends OperationFunction def finalizeF = new BsonJavaScript('function(key, value){}') def mapF = new BsonJavaScript('function(){ }') def reduceF = new BsonJavaScript('function(key, values){ }') - def operation = new MapReduceWithInlineResultsOperation(helper.namespace, mapF, reduceF, documentCodec) + def operation = new MapReduceWithInlineResultsOperation(helper.namespace, mapF, reduceF, bsonDocumentCodec) .filter(filter) .finalizeFunction(finalizeF) .scope(scope) @@ -129,7 +130,7 @@ class MapReduceWithInlineResultsOperationSpecification extends OperationFunction def operation = mapReduceOperation when: - def results = executeAndCollectBatchCursorResults(operation, async) + def results = executeAndCollectBatchCursorResults(operation, async) as Set then: results == expectedResults @@ -140,8 +141,8 @@ class MapReduceWithInlineResultsOperationSpecification extends OperationFunction def 'should use the ReadBindings readPreference to set slaveOK'() { when: - def operation = new MapReduceWithInlineResultsOperation(helper.namespace, new BsonJavaScript('function(){ }'), - new BsonJavaScript('function(key, values){ }'), documentCodec) + def operation = new MapReduceWithInlineResultsOperation(helper.namespace, new BsonJavaScript('function(){ }'), + new BsonJavaScript('function(key, values){ }'), bsonDocumentCodec) then: testOperationSlaveOk(operation, [3, 4, 0], readPreference, async, helper.commandResult) @@ -152,17 +153,12 @@ class MapReduceWithInlineResultsOperationSpecification extends OperationFunction def 'should create the expected command'() { when: - def operation = new MapReduceWithInlineResultsOperation(helper.namespace, new BsonJavaScript('function(){ }'), - new BsonJavaScript('function(key, values){ }'), documentCodec) + def operation = new MapReduceWithInlineResultsOperation(helper.namespace, new BsonJavaScript('function(){ }'), + new BsonJavaScript('function(key, values){ }'), bsonDocumentCodec) def expectedCommand = new BsonDocument('mapreduce', new BsonString(helper.namespace.getCollectionName())) .append('map', operation.getMapFunction()) .append('reduce', operation.getReduceFunction()) .append('out', new BsonDocument('inline', new BsonInt32(1))) - .append('query', new BsonNull()) - .append('sort', new BsonNull()) - .append('finalize', new BsonNull()) - .append('scope', new BsonNull()) - .append('verbose', BsonBoolean.FALSE) then: testOperation(operation, serverVersion, expectedCommand, async, helper.commandResult) @@ -205,8 +201,8 @@ class MapReduceWithInlineResultsOperationSpecification extends OperationFunction def 'should throw an exception when using an unsupported ReadConcern'() { given: - def operation = new MapReduceWithInlineResultsOperation(helper.namespace, new BsonJavaScript('function(){ }'), - new BsonJavaScript('function(key, values){ }'), documentCodec) + def operation = new MapReduceWithInlineResultsOperation(helper.namespace, new BsonJavaScript('function(){ }'), + new BsonJavaScript('function(key, values){ }'), bsonDocumentCodec) when: testOperationThrows(operation, [3, 0, 0], readConcern, async) @@ -221,8 +217,8 @@ class MapReduceWithInlineResultsOperationSpecification extends OperationFunction def 'should throw an exception when using an unsupported Collation'() { given: - def operation = new MapReduceWithInlineResultsOperation(helper.namespace, new BsonJavaScript('function(){ }'), - new BsonJavaScript('function(key, values){ }'), documentCodec).collation(defaultCollation) + def operation = new MapReduceWithInlineResultsOperation(helper.namespace, new BsonJavaScript('function(){ }'), + new BsonJavaScript('function(key, values){ }'), bsonDocumentCodec).collation(defaultCollation) when: testOperationThrows(operation, [3, 2, 0], async) @@ -240,11 +236,11 @@ class MapReduceWithInlineResultsOperationSpecification extends OperationFunction given: def document = Document.parse('{_id: 1, str: "foo"}') getCollectionHelper().insertDocuments(document) - def operation = new MapReduceWithInlineResultsOperation( + def operation = new MapReduceWithInlineResultsOperation( namespace, - new BsonJavaScript('function(){ emit( this._id, this.str ); }'), - new BsonJavaScript('function(key, values){ return key, values; }'), - documentCodec) + new BsonJavaScript('function(){ emit( this.str, 1 ); }'), + new BsonJavaScript('function(key, values){ return Array.sum(values); }'), + bsonDocumentCodec) .filter(BsonDocument.parse('{str: "FOO"}')) .collation(caseInsensitiveCollation) @@ -252,7 +248,7 @@ class MapReduceWithInlineResultsOperationSpecification extends OperationFunction def results = executeAndCollectBatchCursorResults(operation, async) then: - results == [Document.parse('{_id: 1.0, value: "foo"}')] + results == [new BsonDocument('_id', new BsonString('foo')).append('value', new BsonDouble(1))] where: async << [true, false] @@ -272,17 +268,12 @@ class MapReduceWithInlineResultsOperationSpecification extends OperationFunction { "mapreduce" : "coll", "map" : { "$code" : "function(){ }" }, "reduce" : { "$code" : "function(key, values){ }" }, - "out" : { "inline" : 1 }, - "query" : null, - "sort" : null, - "finalize" : null, - "scope" : null, - "verbose" : false, + "out" : { "inline" : 1 } }''') appendReadConcernToCommand(sessionContext, commandDocument) - def operation = new MapReduceWithInlineResultsOperation(helper.namespace, new BsonJavaScript('function(){ }'), - new BsonJavaScript('function(key, values){ }'), documentCodec) + def operation = new MapReduceWithInlineResultsOperation(helper.namespace, new BsonJavaScript('function(){ }'), + new BsonJavaScript('function(key, values){ }'), bsonDocumentCodec) when: operation.execute(binding) @@ -324,17 +315,12 @@ class MapReduceWithInlineResultsOperationSpecification extends OperationFunction { "mapreduce" : "coll", "map" : { "$code" : "function(){ }" }, "reduce" : { "$code" : "function(key, values){ }" }, - "out" : { "inline" : 1 }, - "query" : null, - "sort" : null, - "finalize" : null, - "scope" : null, - "verbose" : false, + "out" : { "inline" : 1 } }''') appendReadConcernToCommand(sessionContext, commandDocument) - def operation = new MapReduceWithInlineResultsOperation(helper.namespace, new BsonJavaScript('function(){ }'), - new BsonJavaScript('function(key, values){ }'), documentCodec) + def operation = new MapReduceWithInlineResultsOperation(helper.namespace, new BsonJavaScript('function(){ }'), + new BsonJavaScript('function(key, values){ }'), bsonDocumentCodec) when: executeAsync(operation, binding) diff --git a/driver-legacy/src/test/functional/com/mongodb/MapReduceTest.java b/driver-legacy/src/test/functional/com/mongodb/MapReduceTest.java index 994fd79bca6..471da5f3955 100644 --- a/driver-legacy/src/test/functional/com/mongodb/MapReduceTest.java +++ b/driver-legacy/src/test/functional/com/mongodb/MapReduceTest.java @@ -30,6 +30,7 @@ import static com.mongodb.ClusterFixture.isDiscoverableReplicaSet; import static com.mongodb.ClusterFixture.isSharded; import static com.mongodb.ClusterFixture.serverVersionAtLeast; +import static com.mongodb.ClusterFixture.serverVersionLessThan; import static com.mongodb.DBObjectMatchers.hasFields; import static com.mongodb.DBObjectMatchers.hasSubdocument; import static java.util.concurrent.TimeUnit.SECONDS; @@ -150,6 +151,7 @@ public void testMapReduceWithOutputToAnotherDatabase() { MapReduceCommand.OutputType.REPLACE, new BasicDBObject()); command.setOutputDB(MR_DATABASE); + getClient().getDatabase(MR_DATABASE).createCollection(DEFAULT_COLLECTION); MapReduceOutput output = collection.mapReduce(command); @@ -311,9 +313,15 @@ public void shouldReturnStatisticsForInlineMapReduce() { //then //duration is not working on the unstable server version // assertThat(output.getDuration(), is(greaterThan(0))); - assertThat(output.getEmitCount(), is(6)); - assertThat(output.getInputCount(), is(3)); - assertThat(output.getOutputCount(), is(4)); + if (serverVersionLessThan("4.4")) { + assertThat(output.getEmitCount(), is(6)); + assertThat(output.getInputCount(), is(3)); + assertThat(output.getOutputCount(), is(4)); + } else { + assertThat(output.getEmitCount(), is(0)); + assertThat(output.getInputCount(), is(0)); + assertThat(output.getOutputCount(), is(0)); + } } @Test @@ -329,10 +337,17 @@ public void shouldReturnStatisticsForMapReduceIntoACollection() { MapReduceOutput output = collection.mapReduce(command); //then - assertThat(output.getDuration(), is(greaterThanOrEqualTo(0))); - assertThat(output.getEmitCount(), is(6)); - assertThat(output.getInputCount(), is(3)); - assertThat(output.getOutputCount(), is(4)); + if (serverVersionLessThan("4.4")) { + assertThat(output.getDuration(), is(greaterThanOrEqualTo(0))); + assertThat(output.getEmitCount(), is(6)); + assertThat(output.getInputCount(), is(3)); + assertThat(output.getOutputCount(), is(4)); + } else { + assertThat(output.getDuration(), is(0)); + assertThat(output.getEmitCount(), is(0)); + assertThat(output.getInputCount(), is(0)); + assertThat(output.getOutputCount(), is(0)); + } } diff --git a/driver-sync/src/test/functional/com/mongodb/client/MongoCollectionTest.java b/driver-sync/src/test/functional/com/mongodb/client/MongoCollectionTest.java index ae8f0357c43..2421a02633c 100644 --- a/driver-sync/src/test/functional/com/mongodb/client/MongoCollectionTest.java +++ b/driver-sync/src/test/functional/com/mongodb/client/MongoCollectionTest.java @@ -39,6 +39,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; public class MongoCollectionTest extends DatabaseTestCase { @@ -135,8 +136,8 @@ public void testMapReduceWithGenerics() { List result = collection.mapReduce(mapFunction, reduceFunction, Name.class).into(new ArrayList()); // then - assertEquals(new Name("Pete", 2), result.get(0)); - assertEquals(new Name("Sam", 1), result.get(1)); + assertTrue(result.contains(new Name("Pete", 2))); + assertTrue(result.contains(new Name("Sam", 1))); } @Test From f58395a071a8da25e56628babf7df29d405eb2b9 Mon Sep 17 00:00:00 2001 From: Jeff Yemin Date: Wed, 27 Jan 2021 19:58:01 -0500 Subject: [PATCH 07/13] Fix metaTextScore functional tests Change the test so that it produces a non-zero score so that the score won't be excluded by later server versions. JAVA-3942 --- .../ProjectionFunctionalSpecification.groovy | 35 ++++++++++--------- .../model/SortsFunctionalSpecification.groovy | 15 +++++--- 2 files changed, 28 insertions(+), 22 deletions(-) diff --git a/driver-core/src/test/functional/com/mongodb/client/model/ProjectionFunctionalSpecification.groovy b/driver-core/src/test/functional/com/mongodb/client/model/ProjectionFunctionalSpecification.groovy index 69738c62002..64dd88fa08e 100644 --- a/driver-core/src/test/functional/com/mongodb/client/model/ProjectionFunctionalSpecification.groovy +++ b/driver-core/src/test/functional/com/mongodb/client/model/ProjectionFunctionalSpecification.groovy @@ -23,6 +23,7 @@ import org.bson.conversions.Bson import static com.mongodb.client.model.Filters.and import static com.mongodb.client.model.Filters.eq +import static com.mongodb.client.model.Filters.text import static com.mongodb.client.model.Projections.elemMatch import static com.mongodb.client.model.Projections.exclude import static com.mongodb.client.model.Projections.excludeId @@ -32,21 +33,21 @@ import static com.mongodb.client.model.Projections.metaTextScore import static com.mongodb.client.model.Projections.slice class ProjectionFunctionalSpecification extends OperationFunctionalSpecification { - def a = new Document('_id', 1).append('x', 1).append('y', [new Document('a', 1).append('b', 2), - new Document('a', 2).append('b', 3), - new Document('a', 3).append('b', 4)]) - def aYSlice1 = new Document('_id', 1).append('x', 1).append('y', [new Document('a', 1).append('b', 2)]) - def aYSlice12 = new Document('_id', 1).append('x', 1).append('y', [new Document('a', 2).append('b', 3), - new Document('a', 3).append('b', 4)]) - def aNoY = new Document('_id', 1).append('x', 1) + def a = new Document('_id', 1).append('x', 'coffee').append('y', [new Document('a', 1).append('b', 2), + new Document('a', 2).append('b', 3), + new Document('a', 3).append('b', 4)]) + def aYSlice1 = new Document('_id', 1).append('x', 'coffee').append('y', [new Document('a', 1).append('b', 2)]) + def aYSlice12 = new Document('_id', 1).append('x', 'coffee').append('y', [new Document('a', 2).append('b', 3), + new Document('a', 3).append('b', 4)]) + def aNoY = new Document('_id', 1).append('x', 'coffee') def aId = new Document('_id', 1) - def aNoId = new Document().append('x', 1).append('y', [new Document('a', 1).append('b', 2), - new Document('a', 2).append('b', 3), - new Document('a', 3).append('b', 4)]) - def aWithScore = new Document('_id', 1).append('x', 1).append('y', [new Document('a', 1).append('b', 2), - new Document('a', 2).append('b', 3), - new Document('a', 3).append('b', 4)]) - .append('score', 0.0) + def aNoId = new Document().append('x', 'coffee').append('y', [new Document('a', 1).append('b', 2), + new Document('a', 2).append('b', 3), + new Document('a', 3).append('b', 4)]) + def aWithScore = new Document('_id', 1).append('x', 'coffee').append('y', [new Document('a', 1).append('b', 2), + new Document('a', 2).append('b', 3), + new Document('a', 3).append('b', 4)]) + .append('score', 1.0) def setup() { getCollectionHelper().insertDocuments(a) @@ -74,7 +75,7 @@ class ProjectionFunctionalSpecification extends OperationFunctionalSpecification find(exclude(['x', 'y', 'x'])) == [aId] } - def 'excludeId'() { + def 'excludeId helper'() { expect: find(excludeId()) == [aNoId] } @@ -98,10 +99,10 @@ class ProjectionFunctionalSpecification extends OperationFunctionalSpecification def 'metaTextScore'() { given: - getCollectionHelper().createIndex(new Document('y', 'text')) + getCollectionHelper().createIndex(new Document('x', 'text')) expect: - find(metaTextScore('score')) == [aWithScore] + find(text('coffee'), metaTextScore('score')) == [aWithScore] } def 'combine fields'() { diff --git a/driver-core/src/test/functional/com/mongodb/client/model/SortsFunctionalSpecification.groovy b/driver-core/src/test/functional/com/mongodb/client/model/SortsFunctionalSpecification.groovy index fd78a97fef5..e7e552da3e4 100644 --- a/driver-core/src/test/functional/com/mongodb/client/model/SortsFunctionalSpecification.groovy +++ b/driver-core/src/test/functional/com/mongodb/client/model/SortsFunctionalSpecification.groovy @@ -27,13 +27,13 @@ import static com.mongodb.client.model.Sorts.orderBy class SortsFunctionalSpecification extends OperationFunctionalSpecification { def a = new Document('_id', 1).append('x', 1) - .append('y', 'b') + .append('y', 'bear') def b = new Document('_id', 2).append('x', 1) - .append('y', 'a') + .append('y', 'albatross') def c = new Document('_id', 3).append('x', 2) - .append('y', 'c') + .append('y', 'cat') def setup() { getCollectionHelper().insertDocuments(a, b, c) @@ -44,7 +44,11 @@ class SortsFunctionalSpecification extends OperationFunctionalSpecification { } def 'find'(Bson sort, Bson projection) { - getCollectionHelper().find(new Document(), sort, projection) + find(new Document(), sort, projection) + } + + def 'find'(Bson filter, Bson sort, Bson projection) { + getCollectionHelper().find(filter, sort, projection) } def 'ascending'() { @@ -66,7 +70,8 @@ class SortsFunctionalSpecification extends OperationFunctionalSpecification { getCollectionHelper().createIndex(new Document('y', 'text')) expect: - find(metaTextScore('score'), new Document('score', new Document('$meta', 'textScore')))*.containsKey('score') + find(new Document('$text', new Document('$search', 'bear')), metaTextScore('score'), + new Document('score', new Document('$meta', 'textScore')))*.containsKey('score') } def 'orderBy'() { From 3c495994c07bd60cff9e9cb8ec339dfde216a703 Mon Sep 17 00:00:00 2001 From: Jeff Yemin Date: Wed, 27 Jan 2021 19:58:14 -0500 Subject: [PATCH 08/13] Add 4.4 to Evergreen matrix JAVA-3942 --- .evergreen/.evg.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.evergreen/.evg.yml b/.evergreen/.evg.yml index 4683cb0e15c..f39628aa3af 100644 --- a/.evergreen/.evg.yml +++ b/.evergreen/.evg.yml @@ -395,6 +395,10 @@ axes: - id: version display_name: MongoDB Version values: + - id: "4.4" + display_name: "4.4" + variables: + VERSION: "4.4" - id: "4.2" display_name: "4.2" variables: @@ -530,14 +534,14 @@ buildvariants: - matrix_name: "tests-jdk6-secure" matrix_spec: { auth: "auth", ssl: "ssl", jdk: "jdk6", version: "*", topology: "*", os: "*" } - exclude_spec: { auth: "auth", ssl: "ssl", jdk: "jdk6", version: ["4.0", "4.2"], topology: "*", os: "*" } + exclude_spec: { auth: "auth", ssl: "ssl", jdk: "jdk6", version: ["4.0", "4.2", "4.4"], topology: "*", os: "*" } display_name: "${version} ${topology} ${auth} ${ssl} ${jdk} ${os} " tags: ["tests-variant"] tasks: - name: "test" - matrix_name: "tests-jdk8-secure" - matrix_spec: { auth: "auth", ssl: "ssl", jdk: "jdk8", version: ["4.0", "4.2"], topology: "*", os: "*" } + matrix_spec: { auth: "auth", ssl: "ssl", jdk: "jdk8", version: ["4.0", "4.2", "4.4"], topology: "*", os: "*" } display_name: "${version} ${topology} ${auth} ${ssl} ${jdk} ${os} " tags: ["tests-variant"] tasks: From 424d17c5f88c892184a1cfc761a874d14ce067f3 Mon Sep 17 00:00:00 2001 From: Jeff Yemin Date: Thu, 28 Jan 2021 09:24:58 -0500 Subject: [PATCH 09/13] Fix upload of Junit test results JAVA-3942 --- .evergreen/.evg.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/.evg.yml b/.evergreen/.evg.yml index f39628aa3af..193204d03e1 100644 --- a/.evergreen/.evg.yml +++ b/.evergreen/.evg.yml @@ -194,7 +194,7 @@ functions: "upload test results": - command: attach.xunit_results params: - file: ./src/*/build/test-results/TEST-*.xml + file: ./src/*/build/test-results/test/TEST-*.xml "bootstrap mongo-orchestration": - command: shell.exec From bddbb54d0e8a98cf86f660d37304404367c4d505 Mon Sep 17 00:00:00 2001 From: Jeff Yemin Date: Thu, 28 Jan 2021 15:58:59 -0500 Subject: [PATCH 10/13] Disable tests of geoHaystack indexes on 5.0+ servers JAVA-3942 --- .../mongodb/client/model/IndexesFunctionalSpecification.groovy | 3 +++ .../operation/CreateIndexesOperationSpecification.groovy | 2 ++ 2 files changed, 5 insertions(+) diff --git a/driver-core/src/test/functional/com/mongodb/client/model/IndexesFunctionalSpecification.groovy b/driver-core/src/test/functional/com/mongodb/client/model/IndexesFunctionalSpecification.groovy index cc1f05ce364..7a33ccf3413 100644 --- a/driver-core/src/test/functional/com/mongodb/client/model/IndexesFunctionalSpecification.groovy +++ b/driver-core/src/test/functional/com/mongodb/client/model/IndexesFunctionalSpecification.groovy @@ -17,7 +17,9 @@ package com.mongodb.client.model import com.mongodb.OperationFunctionalSpecification +import spock.lang.IgnoreIf +import static com.mongodb.ClusterFixture.serverVersionGreaterThan import static com.mongodb.client.model.Indexes.ascending import static com.mongodb.client.model.Indexes.compoundIndex import static com.mongodb.client.model.Indexes.descending @@ -98,6 +100,7 @@ class IndexesFunctionalSpecification extends OperationFunctionalSpecification { getCollectionHelper().listIndexes()*.get('key').contains(parse('{x : "2d"}')) } + @IgnoreIf({ serverVersionGreaterThan('4.4') }) def 'geoHaystack'() { when: getCollectionHelper().createIndex(geoHaystack('x', descending('b')), 2.0) diff --git a/driver-core/src/test/functional/com/mongodb/operation/CreateIndexesOperationSpecification.groovy b/driver-core/src/test/functional/com/mongodb/operation/CreateIndexesOperationSpecification.groovy index cfbab9cb7fe..df0440c27a6 100644 --- a/driver-core/src/test/functional/com/mongodb/operation/CreateIndexesOperationSpecification.groovy +++ b/driver-core/src/test/functional/com/mongodb/operation/CreateIndexesOperationSpecification.groovy @@ -38,6 +38,7 @@ import static com.mongodb.ClusterFixture.getBinding import static com.mongodb.ClusterFixture.isDiscoverableReplicaSet import static com.mongodb.ClusterFixture.isSharded import static com.mongodb.ClusterFixture.serverVersionAtLeast +import static com.mongodb.ClusterFixture.serverVersionGreaterThan import static java.util.concurrent.TimeUnit.SECONDS class CreateIndexesOperationSpecification extends OperationFunctionalSpecification { @@ -290,6 +291,7 @@ class CreateIndexesOperationSpecification extends OperationFunctionalSpecificati async << [true, false] } + @IgnoreIf({ serverVersionGreaterThan('4.4') }) def 'should be able to create a geoHaystack indexes'() { given: def operation = new CreateIndexesOperation(getNamespace(), From 34c7f43b3cad86fd03b681307297713b7dcb8c16 Mon Sep 17 00:00:00 2001 From: Jeff Yemin Date: Thu, 28 Jan 2021 16:57:16 -0500 Subject: [PATCH 11/13] Disable explain test for sharded clusters for 5.0 and up JAVA-3942 --- .../com/mongodb/operation/FindOperationSpecification.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/driver-core/src/test/functional/com/mongodb/operation/FindOperationSpecification.groovy b/driver-core/src/test/functional/com/mongodb/operation/FindOperationSpecification.groovy index 259fbdd9597..b868fb1fe05 100644 --- a/driver-core/src/test/functional/com/mongodb/operation/FindOperationSpecification.groovy +++ b/driver-core/src/test/functional/com/mongodb/operation/FindOperationSpecification.groovy @@ -61,6 +61,7 @@ import static com.mongodb.ClusterFixture.getBinding import static com.mongodb.ClusterFixture.getCluster import static com.mongodb.ClusterFixture.isSharded import static com.mongodb.ClusterFixture.serverVersionAtLeast +import static com.mongodb.ClusterFixture.serverVersionGreaterThan import static com.mongodb.CursorType.NonTailable import static com.mongodb.CursorType.Tailable import static com.mongodb.CursorType.TailableAwait @@ -682,7 +683,7 @@ class FindOperationSpecification extends OperationFunctionalSpecification { ].combinations() } - + @IgnoreIf({ serverVersionGreaterThan("4.4") }) def 'should explain with $explain modifier'() { given: def operation = new FindOperation(getNamespace(), new BsonDocumentCodec()) From 5a4de01a40e4ee8748047fc6b4e3df6831e40876 Mon Sep 17 00:00:00 2001 From: Jeff Yemin Date: Thu, 28 Jan 2021 17:20:40 -0500 Subject: [PATCH 12/13] Add "latest" to Evergreen matrix JAVA-3942 --- .evergreen/.evg.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.evergreen/.evg.yml b/.evergreen/.evg.yml index 193204d03e1..4f2b00d0432 100644 --- a/.evergreen/.evg.yml +++ b/.evergreen/.evg.yml @@ -395,6 +395,10 @@ axes: - id: version display_name: MongoDB Version values: + - id: "latest" + display_name: "latest" + variables: + VERSION: "latest" - id: "4.4" display_name: "4.4" variables: @@ -534,14 +538,14 @@ buildvariants: - matrix_name: "tests-jdk6-secure" matrix_spec: { auth: "auth", ssl: "ssl", jdk: "jdk6", version: "*", topology: "*", os: "*" } - exclude_spec: { auth: "auth", ssl: "ssl", jdk: "jdk6", version: ["4.0", "4.2", "4.4"], topology: "*", os: "*" } + exclude_spec: { auth: "auth", ssl: "ssl", jdk: "jdk6", version: ["4.0", "4.2", "4.4", "latest"], topology: "*", os: "*" } display_name: "${version} ${topology} ${auth} ${ssl} ${jdk} ${os} " tags: ["tests-variant"] tasks: - name: "test" - matrix_name: "tests-jdk8-secure" - matrix_spec: { auth: "auth", ssl: "ssl", jdk: "jdk8", version: ["4.0", "4.2", "4.4"], topology: "*", os: "*" } + matrix_spec: { auth: "auth", ssl: "ssl", jdk: "jdk8", version: ["4.0", "4.2", "4.4", "latest"], topology: "*", os: "*" } display_name: "${version} ${topology} ${auth} ${ssl} ${jdk} ${os} " tags: ["tests-variant"] tasks: From cfd33c61148e6babe43bff1a2531e4384c25ff34 Mon Sep 17 00:00:00 2001 From: Jeff Yemin Date: Thu, 28 Jan 2021 14:30:26 -0500 Subject: [PATCH 13/13] Add COMPATIBILITY.MD JAVA-3942 --- COMPATIBILITY.MD | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 COMPATIBILITY.MD diff --git a/COMPATIBILITY.MD b/COMPATIBILITY.MD new file mode 100644 index 00000000000..6cfaca95d6b --- /dev/null +++ b/COMPATIBILITY.MD @@ -0,0 +1,19 @@ +### Incompatibilities with MongoDB 4.2 + +MongoDB 4.2 introduced the "NonResumableChangeStreamError" error label, which is not recognized +by the 4.0-era Java driver. This may cause the 4.0-era driver to resume a change stream in cases +where the server has indicated that it should not. This may cause iteration of a change stream to +loop infinitely instead of reporting an error to the application + +### Incompatibilities with MongoDB 4.4 + +MongoDB 4.4 introduced the "ResumableChangeStreamError" error label, which is not recognized +by the 4.0-era Java driver. This may cause the 4.0-era driver to not resume a change stream in cases +where the server has indicated that it should. + +MongoDB 4.4 removed support for mapReduce statistics. The 4.0-era driver assumes statistics are always +included in the reply, so any use of mapReduce with the 4.0-era driver will fail. MongoDB 4.4 also +removed support for the default values of sharded and nonAtomic fields for mapReduce to a collection. +The 4.0-era driver always includes the default values of these fields (even if unset by the application), +so this will cause any mapReduce with $out to fail as well. +