3.8.x continuous matrix testing

.evergreen/.evg.yml

@@ -194,7 +194,7 @@ functions:
   "upload test results":
     - command: attach.xunit_results
       params:
-        file: ./src/*/build/test-results/TEST-*.xml
+        file: ./src/*/build/test-results/test/TEST-*.xml
 
   "bootstrap mongo-orchestration":
     - command: shell.exec
@@ -316,15 +316,6 @@ functions:
           ${PREPARE_SHELL}
           echo '{"results": [{ "status": "FAIL", "test_file": "Build", "log_raw": "No test-results.json found was created"  } ]}' > ${PROJECT_DIRECTORY}/test-results.json
 
-  "install dependencies":
-    type: test
-    params:
-      working_dir: "src"
-      script: |
-        ${PREPARE_SHELL}
-        file="${PROJECT_DIRECTORY}/.evergreen/install-dependencies.sh"
-        [ -f ${file} ] && sh ${file} || echo "${file} not available, skipping"
-
 # Anchors
 
 hosts: &hosts
@@ -337,7 +328,6 @@ pre:
   - func: "fix absolute paths"
   - func: "init test-results"
   - func: "make files executable"
-  - func: "install dependencies"
 
 post:
   # Removed, causing timeouts
@@ -357,9 +347,6 @@ tasks:
         - func: "upload build"
 
     - name: "test"
-      depends_on:
-        - variant: "static-checks"
-          name: "static-analysis"
       commands:
         - func: "bootstrap mongo-orchestration"
         - func: "run tests"
@@ -408,6 +395,18 @@ axes:
   - id: version
     display_name: MongoDB Version
     values:
+      - id: "latest"
+        display_name: "latest"
+        variables:
+          VERSION: "latest"
+      - id: "4.4"
+        display_name: "4.4"
+        variables:
+          VERSION: "4.4"
+      - id: "4.2"
+        display_name: "4.2"
+        variables:
+          VERSION: "4.2"
       - id: "4.0"
         display_name: "4.0"
         variables:
@@ -539,14 +538,14 @@ buildvariants:
 
 - matrix_name: "tests-jdk6-secure"
   matrix_spec:  { auth: "auth", ssl: "ssl", jdk: "jdk6", version: "*", topology: "*", os: "*" }
-  exclude_spec: { auth: "auth", ssl: "ssl", jdk: "jdk6", version: ["4.0"], topology: "*", os: "*" }
+  exclude_spec: { auth: "auth", ssl: "ssl", jdk: "jdk6", version: ["4.0", "4.2", "4.4", "latest"], topology: "*", os: "*" }
   display_name: "${version} ${topology} ${auth} ${ssl} ${jdk} ${os} "
   tags: ["tests-variant"]
   tasks:
      - name: "test"
 
 - matrix_name: "tests-jdk8-secure"
-  matrix_spec: { auth: "auth", ssl: "ssl", jdk: "jdk8", version: ["4.0"], topology: "*", os: "*" }
+  matrix_spec: { auth: "auth", ssl: "ssl", jdk: "jdk8", version: ["4.0", "4.2", "4.4", "latest"], topology: "*", os: "*" }
   display_name: "${version} ${topology} ${auth} ${ssl} ${jdk} ${os} "
   tags: ["tests-variant"]
   tasks:

COMPATIBILITY.MD

@@ -0,0 +1,19 @@
+### Incompatibilities with MongoDB 4.2
+
+MongoDB 4.2 introduced the "NonResumableChangeStreamError" error label, which is not recognized
+by the 4.0-era Java driver.  This may cause the 4.0-era driver to resume a change stream in cases
+where the server has indicated that it should not. This may cause iteration of a change stream to 
+loop infinitely instead of reporting an error to the application
+
+### Incompatibilities with MongoDB 4.4
+
+MongoDB 4.4 introduced the "ResumableChangeStreamError" error label, which is not recognized
+by the 4.0-era Java driver. This may cause the 4.0-era driver to not resume a change stream in cases
+where the server has indicated that it should.  
+
+MongoDB 4.4 removed support for mapReduce statistics.  The 4.0-era driver assumes statistics are always
+included in the reply, so any use of mapReduce with the 4.0-era driver will fail.  MongoDB 4.4 also 
+removed support for the default values of sharded and nonAtomic fields for mapReduce to a collection.
+The 4.0-era driver always includes the default values of these fields (even if unset by the application), 
+so this will cause any mapReduce with $out to fail as well.
+

driver-core/src/main/com/mongodb/internal/operation/ServerVersionHelper.java

@@ -45,6 +45,10 @@ public static boolean serverIsAtLeastVersionFourDotZero(final ConnectionDescript
         return serverIsAtLeastVersion(description, new ServerVersion(4, 0));
     }
 
+    public static boolean serverIsAtLeastVersionFourDotFour(final ConnectionDescription description) {
+        return serverIsAtLeastVersion(description, new ServerVersion(4, 4));
+    }
+
     private static boolean serverIsAtLeastVersion(final ConnectionDescription description, final ServerVersion serverVersion) {
         return description.getServerVersion().compareTo(serverVersion) >= 0;
     }

driver-core/src/main/com/mongodb/operation/MapReduceHelper.java

@@ -17,6 +17,7 @@
 package com.mongodb.operation;
 
 import org.bson.BsonDocument;
+import org.bson.BsonInt32;
 
 final class MapReduceHelper {
 
@@ -26,19 +27,19 @@ static MapReduceStatistics createStatistics(final BsonDocument result) {
     }
 
     private static int getInputCount(final BsonDocument result) {
-        return result.getDocument("counts").getNumber("input").intValue();
+        return result.getDocument("counts", new BsonDocument()).getNumber("input", new BsonInt32(0)).intValue();
     }
 
     private static int getOutputCount(final BsonDocument result) {
-        return result.getDocument("counts").getNumber("output").intValue();
+        return result.getDocument("counts", new BsonDocument()).getNumber("output", new BsonInt32(0)).intValue();
     }
 
     private static int getEmitCount(final BsonDocument result) {
-        return result.getDocument("counts").getNumber("emit").intValue();
+        return result.getDocument("counts", new BsonDocument()).getNumber("emit", new BsonInt32(0)).intValue();
     }
 
     private static int getDuration(final BsonDocument result) {
-        return result.getNumber("timeMillis").intValue();
+        return result.getNumber("timeMillis", new BsonInt32(0)).intValue();
     }
 
     private MapReduceHelper() {

driver-core/src/main/com/mongodb/operation/MapReduceToCollectionOperation.java

@@ -42,8 +42,10 @@
 import static com.mongodb.assertions.Assertions.isTrue;
 import static com.mongodb.assertions.Assertions.notNull;
 import static com.mongodb.internal.async.ErrorHandlingResultCallback.errorHandlingCallback;
+import static com.mongodb.internal.operation.ServerVersionHelper.serverIsAtLeastVersionFourDotFour;
 import static com.mongodb.operation.CommandOperationHelper.executeWrappedCommandProtocol;
 import static com.mongodb.operation.CommandOperationHelper.executeWrappedCommandProtocolAsync;
+import static com.mongodb.operation.DocumentHelper.putIfNotNull;
 import static com.mongodb.operation.DocumentHelper.putIfNotZero;
 import static com.mongodb.operation.DocumentHelper.putIfTrue;
 import static com.mongodb.operation.OperationHelper.AsyncCallableWithConnection;
@@ -581,21 +583,23 @@ public MapReduceStatistics apply(final BsonDocument result, final ServerAddress
 
     private BsonDocument getCommand(final ConnectionDescription description) {
         BsonDocument outputDocument = new BsonDocument(getAction(), new BsonString(getCollectionName()));
-        outputDocument.append("sharded", BsonBoolean.valueOf(isSharded()));
-        outputDocument.append("nonAtomic", BsonBoolean.valueOf(isNonAtomic()));
+        if (description != null && !serverIsAtLeastVersionFourDotFour(description)) {
+            putIfTrue(outputDocument, "sharded", isSharded());
+            putIfTrue(outputDocument, "nonAtomic", isNonAtomic());
+        }
         if (getDatabaseName() != null) {
             outputDocument.put("db", new BsonString(getDatabaseName()));
         }
         BsonDocument commandDocument = new BsonDocument("mapreduce", new BsonString(namespace.getCollectionName()))
-                                           .append("map", getMapFunction())
-                                           .append("reduce", getReduceFunction())
-                                           .append("out", outputDocument)
-                                           .append("query", asValueOrNull(getFilter()))
-                                           .append("sort", asValueOrNull(getSort()))
-                                           .append("finalize", asValueOrNull(getFinalizeFunction()))
-                                           .append("scope", asValueOrNull(getScope()))
-                                           .append("verbose", BsonBoolean.valueOf(isVerbose()));
-        putIfNotZero(commandDocument, "limit", getLimit());
+                .append("map", getMapFunction())
+                .append("reduce", getReduceFunction())
+                .append("out", outputDocument);
+
+        putIfNotNull(commandDocument, "query", getFilter());
+        putIfNotNull(commandDocument, "sort", getSort());
+        putIfNotNull(commandDocument, "finalize", getFinalizeFunction());
+        putIfNotNull(commandDocument, "scope", getScope());
+        putIfTrue(commandDocument, "verbose", isVerbose());      putIfNotZero(commandDocument, "limit", getLimit());
         putIfNotZero(commandDocument, "maxTimeMS", getMaxTime(MILLISECONDS));
         putIfTrue(commandDocument, "jsMode", isJsMode());
         if (bypassDocumentValidation != null && description != null && serverIsAtLeastVersionThreeDotTwo(description)) {

driver-core/src/main/com/mongodb/operation/MapReduceWithInlineResultsOperation.java

@@ -48,6 +48,7 @@
 import static com.mongodb.internal.async.ErrorHandlingResultCallback.errorHandlingCallback;
 import static com.mongodb.operation.CommandOperationHelper.executeWrappedCommandProtocol;
 import static com.mongodb.operation.CommandOperationHelper.executeWrappedCommandProtocolAsync;
+import static com.mongodb.operation.DocumentHelper.putIfNotNull;
 import static com.mongodb.operation.DocumentHelper.putIfNotZero;
 import static com.mongodb.operation.DocumentHelper.putIfTrue;
 import static com.mongodb.operation.ExplainHelper.asExplainCommand;
@@ -447,14 +448,15 @@ public MapReduceAsyncBatchCursor<T> apply(final BsonDocument result, final Serve
 
     private BsonDocument getCommand(final SessionContext sessionContext) {
         BsonDocument commandDocument = new BsonDocument("mapreduce", new BsonString(namespace.getCollectionName()))
-                                           .append("map", getMapFunction())
-                                           .append("reduce", getReduceFunction())
-                                           .append("out", new BsonDocument("inline", new BsonInt32(1)))
-                                           .append("query", asValueOrNull(getFilter()))
-                                           .append("sort", asValueOrNull(getSort()))
-                                           .append("finalize", asValueOrNull(getFinalizeFunction()))
-                                           .append("scope", asValueOrNull(getScope()))
-                                           .append("verbose", BsonBoolean.valueOf(isVerbose()));
+                .append("map", getMapFunction())
+                .append("reduce", getReduceFunction())
+                .append("out", new BsonDocument("inline", new BsonInt32(1)));
+
+        putIfNotNull(commandDocument, "query", getFilter());
+        putIfNotNull(commandDocument, "sort", getSort());
+        putIfNotNull(commandDocument, "finalize", getFinalizeFunction());
+        putIfNotNull(commandDocument, "scope", getScope());
+        putIfTrue(commandDocument, "verbose", isVerbose());
 
         appendReadConcernToCommand(sessionContext, commandDocument);
         putIfNotZero(commandDocument, "limit", getLimit());

driver-core/src/test/functional/com/mongodb/client/CommandMonitoringTestHelper.java

@@ -193,9 +193,12 @@ private static CommandSucceededEvent massageActualCommandSucceededEvent(final Co
             if (response.containsKey("writeErrors")) {
                 for (BsonValue bsonValue : response.getArray("writeErrors")) {
                     BsonDocument cur = bsonValue.asDocument();
-                    cur.put("code", new BsonInt32(42));
-                    cur.put("errmsg", new BsonString(""));
-                    cur.remove("codeName");
+                    BsonDocument newWriteErrorDocument =
+                            new BsonDocument().append("index", cur.get("index"))
+                                    .append("code", new BsonInt32(42))
+                                    .append("errmsg", new BsonString(""));
+                    cur.clear();
+                    cur.putAll(newWriteErrorDocument);
                 }
             }
             if (actual.getCommandName().equals("update")) {

driver-core/src/test/functional/com/mongodb/client/model/AggregatesFunctionalSpecification.groovy

@@ -674,31 +674,27 @@ class AggregatesFunctionalSpecification extends OperationFunctionalSpecification
         helper.drop()
 
         helper.insertDocuments(Document.parse('{_id: 0, x: 1}'))
-        helper.insertDocuments(Document.parse('{_id: 1, x: 2}'))
         helper.insertDocuments(Document.parse('{_id: 2, x: 1}'))
         helper.insertDocuments(Document.parse('{_id: 3, x: 0}'))
 
         def results = helper.aggregate([sortByCount('$x')])
 
         then:
         results == [Document.parse('{_id: 1, count: 2}'),
-                    Document.parse('{_id: 0, count: 1}'),
-                    Document.parse('{_id: 2, count: 1}')]
+                    Document.parse('{_id: 0, count: 1}')]
 
         when:
         helper.drop()
 
         helper.insertDocuments(Document.parse('{_id: 0, x: 1.4}'))
-        helper.insertDocuments(Document.parse('{_id: 1, x: 2.3}'))
         helper.insertDocuments(Document.parse('{_id: 2, x: 1.1}'))
         helper.insertDocuments(Document.parse('{_id: 3, x: 0.5}'))
 
         results = helper.aggregate([sortByCount(new Document('$floor', '$x'))])
 
         then:
         results == [Document.parse('{_id: 1, count: 2}'),
-                    Document.parse('{_id: 0, count: 1}'),
-                    Document.parse('{_id: 2, count: 1}')]
+                    Document.parse('{_id: 0, count: 1}')]
 
         cleanup:
         helper?.drop()

driver-core/src/test/functional/com/mongodb/client/model/IndexesFunctionalSpecification.groovy

@@ -17,7 +17,9 @@
 package com.mongodb.client.model
 
 import com.mongodb.OperationFunctionalSpecification
+import spock.lang.IgnoreIf
 
+import static com.mongodb.ClusterFixture.serverVersionGreaterThan
 import static com.mongodb.client.model.Indexes.ascending
 import static com.mongodb.client.model.Indexes.compoundIndex
 import static com.mongodb.client.model.Indexes.descending
@@ -98,6 +100,7 @@ class IndexesFunctionalSpecification extends OperationFunctionalSpecification {
         getCollectionHelper().listIndexes()*.get('key').contains(parse('{x : "2d"}'))
     }
 
+    @IgnoreIf({ serverVersionGreaterThan('4.4') })
     def 'geoHaystack'() {
         when:
         getCollectionHelper().createIndex(geoHaystack('x', descending('b')), 2.0)

driver-core/src/test/functional/com/mongodb/client/model/ProjectionFunctionalSpecification.groovy

@@ -23,6 +23,7 @@ import org.bson.conversions.Bson
 
 import static com.mongodb.client.model.Filters.and
 import static com.mongodb.client.model.Filters.eq
+import static com.mongodb.client.model.Filters.text
 import static com.mongodb.client.model.Projections.elemMatch
 import static com.mongodb.client.model.Projections.exclude
 import static com.mongodb.client.model.Projections.excludeId
@@ -32,21 +33,21 @@ import static com.mongodb.client.model.Projections.metaTextScore
 import static com.mongodb.client.model.Projections.slice
 
 class ProjectionFunctionalSpecification extends OperationFunctionalSpecification {
-    def a = new Document('_id', 1).append('x', 1).append('y', [new Document('a', 1).append('b', 2),
-                                                               new Document('a', 2).append('b', 3),
-                                                               new Document('a', 3).append('b', 4)])
-    def aYSlice1 = new Document('_id', 1).append('x', 1).append('y', [new Document('a', 1).append('b', 2)])
-    def aYSlice12 = new Document('_id', 1).append('x', 1).append('y', [new Document('a', 2).append('b', 3),
-                                                                       new Document('a', 3).append('b', 4)])
-    def aNoY = new Document('_id', 1).append('x', 1)
+    def a = new Document('_id', 1).append('x', 'coffee').append('y', [new Document('a', 1).append('b', 2),
+                                                                      new Document('a', 2).append('b', 3),
+                                                                      new Document('a', 3).append('b', 4)])
+    def aYSlice1 = new Document('_id', 1).append('x', 'coffee').append('y', [new Document('a', 1).append('b', 2)])
+    def aYSlice12 = new Document('_id', 1).append('x', 'coffee').append('y', [new Document('a', 2).append('b', 3),
+                                                                              new Document('a', 3).append('b', 4)])
+    def aNoY = new Document('_id', 1).append('x', 'coffee')
     def aId = new Document('_id', 1)
-    def aNoId = new Document().append('x', 1).append('y', [new Document('a', 1).append('b', 2),
-                                                           new Document('a', 2).append('b', 3),
-                                                           new Document('a', 3).append('b', 4)])
-    def aWithScore = new Document('_id', 1).append('x', 1).append('y', [new Document('a', 1).append('b', 2),
-                                                                        new Document('a', 2).append('b', 3),
-                                                                        new Document('a', 3).append('b', 4)])
-                                           .append('score', 0.0)
+    def aNoId = new Document().append('x', 'coffee').append('y', [new Document('a', 1).append('b', 2),
+                                                                  new Document('a', 2).append('b', 3),
+                                                                  new Document('a', 3).append('b', 4)])
+    def aWithScore = new Document('_id', 1).append('x', 'coffee').append('y', [new Document('a', 1).append('b', 2),
+                                                                               new Document('a', 2).append('b', 3),
+                                                                               new Document('a', 3).append('b', 4)])
+            .append('score', 1.0)
 
     def setup() {
         getCollectionHelper().insertDocuments(a)
@@ -74,7 +75,7 @@ class ProjectionFunctionalSpecification extends OperationFunctionalSpecification
         find(exclude(['x', 'y', 'x'])) == [aId]
     }
 
-    def 'excludeId'() {
+    def 'excludeId helper'() {
         expect:
         find(excludeId()) == [aNoId]
     }
@@ -98,10 +99,10 @@ class ProjectionFunctionalSpecification extends OperationFunctionalSpecification
 
     def 'metaTextScore'() {
         given:
-        getCollectionHelper().createIndex(new Document('y', 'text'))
+        getCollectionHelper().createIndex(new Document('x', 'text'))
 
         expect:
-        find(metaTextScore('score')) == [aWithScore]
+        find(text('coffee'), metaTextScore('score')) == [aWithScore]
     }
 
     def 'combine fields'() {

driver-core/src/test/functional/com/mongodb/client/model/SortsFunctionalSpecification.groovy

@@ -27,13 +27,13 @@ import static com.mongodb.client.model.Sorts.orderBy
 
 class SortsFunctionalSpecification extends OperationFunctionalSpecification {
     def a = new Document('_id', 1).append('x', 1)
-                                  .append('y', 'b')
+                                  .append('y', 'bear')
 
     def b = new Document('_id', 2).append('x', 1)
-                                  .append('y', 'a')
+                                  .append('y', 'albatross')
 
     def c = new Document('_id', 3).append('x', 2)
-                                  .append('y', 'c')
+                                  .append('y', 'cat')
 
     def setup() {
         getCollectionHelper().insertDocuments(a, b, c)
@@ -44,7 +44,11 @@ class SortsFunctionalSpecification extends OperationFunctionalSpecification {
     }
 
     def 'find'(Bson sort, Bson projection) {
-        getCollectionHelper().find(new Document(), sort, projection)
+        find(new Document(), sort, projection)
+    }
+
+    def 'find'(Bson filter, Bson sort, Bson projection) {
+        getCollectionHelper().find(filter, sort, projection)
     }
 
     def 'ascending'() {
@@ -66,7 +70,8 @@ class SortsFunctionalSpecification extends OperationFunctionalSpecification {
         getCollectionHelper().createIndex(new Document('y', 'text'))
 
         expect:
-        find(metaTextScore('score'), new Document('score', new Document('$meta', 'textScore')))*.containsKey('score')
+        find(new Document('$text', new Document('$search', 'bear')), metaTextScore('score'),
+                new Document('score', new Document('$meta', 'textScore')))*.containsKey('score')
     }
 
     def 'orderBy'() {