Merge pull request mongodb#318 from igorsol/m3.6.0-rc4

merge r3.6.0-rc4

buildscripts/aws_ec2.py

@@ -44,31 +44,47 @@ def __init__(self):
     @staticmethod
     def wait_for_state(instance, state, wait_time_secs=0, show_progress=False):
         """Wait up to 'wait_time_secs' for instance to be in 'state'.
-           Return True if 'state' reached."""
-        end_time = time.time() + wait_time_secs
+           Return 0 if 'state' reached, 1 otherwise."""
         if show_progress:
             print("Waiting for instance {} to reach '{}' state".format(instance, state),
                   end="",
                   file=sys.stdout)
-        while time.time() < end_time:
+        reached_state = False
+        end_time = time.time() + wait_time_secs
+        while True:
             if show_progress:
                 print(".", end="", file=sys.stdout)
                 sys.stdout.flush()
-            time.sleep(5)
-            instance.load()
-            if instance.state["Name"] == state:
-                if show_progress:
-                    print(" Instance {}!".format(state), file=sys.stdout)
-                    sys.stdout.flush()
-                return True
-        return False
+            try:
+                time_left = end_time - time.time()
+                instance.load()
+                if instance.state["Name"] == state:
+                    reached_state = True
+                    break
+                if time_left <= 0:
+                    break
+            except botocore.exceptions.ClientError:
+                # A ClientError exception can sometimes be generated, due to RequestLimitExceeded,
+                # so we ignore it and retry until we time out.
+                pass
+            wait_interval_secs = 15 if time_left > 15 else time_left
+            time.sleep(wait_interval_secs)
+        if show_progress:
+            if reached_state:
+                print(" Instance {}!".format(instance.state["Name"]), file=sys.stdout)
+            else:
+                print(" Instance in state '{}', failed to reach state '{}'!".format(
+                    instance.state["Name"], state), file=sys.stdout)
+            sys.stdout.flush()
+        return 0 if reached_state else 1
 
     def control_instance(self, mode, image_id, wait_time_secs=0, show_progress=False):
         """Controls an AMI instance. Returns 0 & status information, if successful."""
         if mode not in _MODES:
             raise ValueError(
                 "Invalid mode '{}' specified, choose from {}.".format(mode, _MODES))
 
+        sys.stdout.flush()
         instance = self.connection.Instance(image_id)
         try:
             if mode == "start":
@@ -92,8 +108,9 @@ def control_instance(self, mode, image_id, wait_time_secs=0, show_progress=False
         except botocore.exceptions.ClientError as err:
             return 1, err.message
 
+        ret = 0
         if wait_time_secs > 0:
-            self.wait_for_state(
+            ret = self.wait_for_state(
                 instance=instance,
                 state=state,
                 wait_time_secs=wait_time_secs,
@@ -113,7 +130,7 @@ def control_instance(self, mode, image_id, wait_time_secs=0, show_progress=False
         except botocore.exceptions.ClientError as err:
             return 1, err.message
 
-        return 0, status
+        return ret, status
 
     def tag_instance(self, image_id, tags):
         """Tags an AMI instance. """
@@ -208,7 +225,7 @@ def main():
     control_options.add_option("--waitTimeSecs",
                                dest="wait_time_secs",
                                type=int,
-                               default=60,
+                               default=5 * 60,
                                help="Time to wait for EC2 instance to reach it's new state,"
                                     " defaults to '%default'.")
 

buildscripts/package_test/recipes/install_mongodb.rb

@@ -56,6 +56,12 @@
     command 'apt-get update && apt-get -y -f install'
   end
 
+  # the ubuntu 16.04 image does not have python installed by default
+  # and it is required for the install_compass script
+  execute 'install python' do
+    command 'apt-get install -y python'
+  end
+
   execute 'install mongo shell' do
     command 'dpkg -i `find . -name "*shell*.deb"`'
     cwd homedir

buildscripts/remote_operations.py

@@ -35,6 +35,8 @@
 
 _OPERATIONS = ["shell", "copy_to", "copy_from"]
 
+_SSH_CONNECTION_ERRORS = ["System is booting up.", "Permission denied"]
+
 
 def posix_path(path):
     """ Returns posix path, used on Windows since scp requires posix style paths. """
@@ -100,7 +102,8 @@ def _remote_access(self):
         buff = ""
         while True:
             ret, buff = self._call(cmd)
-            if not ret:
+            # Ignore any connection errors before sshd has fully initialized.
+            if not ret and not any(ssh_error in buff for ssh_error in _SSH_CONNECTION_ERRORS):
                 return ret, buff
             attempt_num += 1
             if attempt_num > self.retries:

buildscripts/resmokeconfig/suites/change_streams_secondary_reads.yml

@@ -0,0 +1,50 @@
+test_kind: js_test
+
+selector:
+  roots:
+  - jstests/change_streams/**/*.js
+  exclude_files:
+  # This test starts a parallel shell which we want to read from the same node as the change stream,
+  # but the parallel shell performs an insert which will not work against the secondary.
+  - jstests/change_streams/only_wake_getmore_for_relevant_changes.js
+  # This test is not expected to work when run against a mongos.
+  - jstests/change_streams/report_latest_observed_oplog_timestamp.js
+
+executor:
+  config:
+    shell_options:
+      global_vars:
+        TestData:
+          defaultReadConcernLevel: majority
+      eval: >-
+        var testingReplication = true;
+        load('jstests/libs/override_methods/set_read_and_write_concerns.js');
+        load('jstests/libs/override_methods/set_read_preference_secondary.js');
+  hooks:
+  - class: ValidateCollections
+  - class: CleanEveryN
+    n: 20
+  fixture:
+    class: ShardedClusterFixture
+    mongos_options:
+      set_parameters:
+        enableTestCommands: 1
+        logComponentVerbosity:
+          verbosity: 0
+          command: 1
+          network:
+            verbosity: 1
+            asio: 2
+          tracking: 0
+    mongod_options:
+      nopreallocj: ''
+      set_parameters:
+        enableTestCommands: 1
+        logComponentVerbosity:
+          verbosity: 0
+          command: 1
+          query: 1
+        numInitialSyncAttempts: 1
+    num_rs_nodes_per_shard: 2
+    enable_sharding:
+    - test

buildscripts/resmokeconfig/suites/jstestfuzz_replication.yml

@@ -31,5 +31,6 @@ executor:
       set_parameters:
         enableTestCommands: 1
         numInitialSyncAttempts: 1
+        writePeriodicNoops: 1
       verbose: ''
     num_nodes: 2

buildscripts/resmokeconfig/suites/jstestfuzz_replication_initsync.yml

@@ -29,5 +29,6 @@ executor:
         enableTestCommands: 1
         numInitialSyncAttempts: 2
         numInitialSyncConnectAttempts: 60
+        writePeriodicNoops: 1
     num_nodes: 2
     start_initial_sync_node: True

buildscripts/resmokeconfig/suites/jstestfuzz_replication_resync.yml

@@ -30,5 +30,6 @@ executor:
         enableTestCommands: 1
         numInitialSyncAttempts: 1
         numInitialSyncConnectAttempts: 60
+        writePeriodicNoops: 1
     num_nodes: 2
     start_initial_sync_node: True

buildscripts/resmokeconfig/suites/jstestfuzz_replication_session.yml

@@ -32,5 +32,6 @@ executor:
       set_parameters:
         enableTestCommands: 1
         numInitialSyncAttempts: 1
+        writePeriodicNoops: 1
       verbose: ''
     num_nodes: 2

buildscripts/resmokeconfig/suites/jstestfuzz_sharded.yml

@@ -24,5 +24,6 @@ executor:
       set_parameters:
         enableTestCommands: 1
         numInitialSyncAttempts: 1
+        writePeriodicNoops: 1
       verbose: ''
     num_shards: 2

buildscripts/resmokeconfig/suites/jstestfuzz_sharded_causal_consistency.yml

@@ -28,5 +28,6 @@ executor:
       set_parameters:
         enableTestCommands: 1
         numInitialSyncAttempts: 1
+        writePeriodicNoops: 1
     num_rs_nodes_per_shard: 2
     num_shards: 2

buildscripts/resmokeconfig/suites/jstestfuzz_sharded_session.yml

@@ -25,5 +25,6 @@ executor:
       set_parameters:
         enableTestCommands: 1
         numInitialSyncAttempts: 1
+        writePeriodicNoops: 1
       verbose: ''
     num_shards: 2

buildscripts/resmokeconfig/suites/read_concern_majority_passthrough.yml

@@ -97,6 +97,7 @@ executor:
       set_parameters:
         enableTestCommands: 1
         numInitialSyncAttempts: 1
+        writePeriodicNoops: 1
       enableMajorityReadConcern: ''
     num_nodes: 2
     # Needs to be set for any ephemeral or no-journaling storage engine

buildscripts/resmokeconfig/suites/retryable_writes_jscore_passthrough.yml

@@ -21,51 +21,14 @@ selector:
   # TODO SERVER-31249: getLastError should not be affected by no-op retries.
   - jstests/core/bulk_legacy_enforce_gle.js
 
-  # TODO SERVER-30532: Add lastErrorObject.upserted to findAndModify no-op retry response.
-  - jstests/core/find_and_modify_server6582.js
-
   # TODO SERVER-31242: findAndModify no-op retry should respect the fields option.
   - jstests/core/crud_api.js
   - jstests/core/find_and_modify2.js
   - jstests/core/find_and_modify_server6865.js
 
-  # TODO SERVER-31243: No-op retry of a findAndModify upsert of a new document should return null
-  # if new=false.
-  - jstests/core/find_and_modify.js
-  - jstests/core/find_and_modify4.js
-  - jstests/core/find_and_modify_empty_coll.js
-  - jstests/core/find_and_modify_empty_update.js
-  - jstests/core/find_and_modify_server6226.js
-
   # TODO SERVER-31245: Inserts to "system.indexes" bypass the check for retryability.
   - jstests/core/batch_write_command_insert.js
 
-  # TODO SERVER-31328: Investigate enabling these workloads once performance of retrying completed
-  # inserts and deletes has improved.
-  #
-  # These tests time out because of too many retryable operations:
-  - jstests/core/bulk_insert.js
-  - jstests/core/insert1.js
-  - jstests/core/remove6.js
-  - jstests/core/removea.js
-
-  # TODO SERVER-29843 / SERVER-31328: Investigate enabling these workloads once performance of
-  # retrying completed writes has improved.
-  #
-  # These tests time out on slower machines because of too many retryable operations:
-  - jstests/core/explain3.js
-  - jstests/core/geo2.js
-  - jstests/core/geo3.js
-  - jstests/core/geo_polygon3.js
-  - jstests/core/geo_s2explain.js
-  - jstests/core/geo_s2sparse.js
-  - jstests/core/geo_s2twofields.js
-  - jstests/core/mr1.js
-  - jstests/core/queryoptimizer3.js
-  - jstests/core/remove9.js
-  - jstests/core/removeb.js
-  - jstests/core/splitvector.js
-
 executor:
   config:
     shell_options:

buildscripts/resmokeconfig/suites/sharding.yml

@@ -4,6 +4,10 @@ selector:
   roots:
   - jstests/sharding/*.js
   exclude_files:
+  # TODO: SERVER-31916
+  - jstests/sharding/aggregation_currentop.js
+  - jstests/sharding/auth_slaveok_routing.js
+  # TODO: SERVER-31937
   - jstests/sharding/kill_sessions.js
 
 executor:

buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml

@@ -145,6 +145,8 @@ selector:
   - jstests/sharding/migration_ignore_interrupts_4.js
   # listCollections is not retryable
   - jstests/sharding/sessions_collection_auto_healing.js
+  # shardCollection is not retryable
+  - jstests/sharding/shard_config_db_collections.js
 
 executor:
   config:

buildscripts/resmokeconfig/suites/sharding_ese.yml

@@ -8,6 +8,10 @@ selector:
   roots:
   - jstests/sharding/*.js
   exclude_files:
+  # TODO: SERVER-31916
+  - jstests/sharding/aggregation_currentop.js
+  - jstests/sharding/auth_slaveok_routing.js
+  # TODO: SERVER-31937
   - jstests/sharding/kill_sessions.js
 
 executor:

buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml

@@ -37,11 +37,12 @@ selector:
   - jstests/sharding/lookup_change_stream_post_image_hashed_shard_key.js
   - jstests/sharding/lookup_change_stream_post_image_compound_shard_key.js
   - jstests/sharding/change_stream_invalidation.js
+  - jstests/sharding/change_stream_lookup_single_shard_cluster.js
   - jstests/sharding/change_stream_remove_shard.js
   - jstests/sharding/change_streams.js
+  - jstests/sharding/change_streams_shards_start_in_sync.js
   - jstests/sharding/change_streams_unsharded_becomes_sharded.js
   - jstests/sharding/close_cursor_on_chunk_migration_to_new_shards.js
-  - jstests/sharding/close_cursor_on_chunk_migration_to_new_shards.js
   - jstests/sharding/enable_sharding_basic.js
   - jstests/sharding/key_rotation.js
   - jstests/sharding/kill_sessions.js

buildscripts/resmokelib/parser.py

@@ -298,7 +298,7 @@ def validate_options(parser, options, args):
         parser.error("Cannot specify both `shellPort` and `shellConnString`")
 
     if options.executor_file:
-        parser.error("--executor is superceded by --suites; specify --suites={} {} to run the"
+        parser.error("--executor is superseded by --suites; specify --suites={} {} to run the"
                      "test(s) under those suite configuration(s)"
                      .format(options.executor_file, " ".join(args)))