multi_stmt_txn_jscore_passthrough_with_migration.yml

test_kind: multi_stmt_txn_passthrough

selector:
  roots:
  - jstests/core/**/*.js
  exclude_files:
  # These tests run in the jscore_txn passthrough suites.
  - jstests/core/txns/**/*.js

  # The following tests fail because a certain command or functionality is not supported by
  # mongos. This command or functionality is placed in a comment next to the failing test.
  - jstests/core/apitest_db.js  # serverStatus output doesn't have storageEngine.
  - jstests/core/apply_ops*.js  # applyOps, SERVER-1439.
  - jstests/core/awaitdata_getmore_cmd.js  # capped collections.
  - jstests/core/bypass_doc_validation.js  # sharded $out output not permitted
  - jstests/core/capped*.js  # capped collections.
  - jstests/core/check_shard_index.js  # checkShardingIndex.
  - jstests/core/collection_truncate.js  # emptycapped.
  - jstests/core/collmod_without_uuid.js # applyOps, SERVER-1439
  - jstests/core/compact_keeps_indexes.js  # compact.
  - jstests/core/currentop.js  # uses fsync.
  - jstests/core/dbadmin.js  # "local" database.
  - jstests/core/dbhash.js  # dbhash.
  - jstests/core/dbhash2.js  # dbhash.
  - jstests/core/diagdata.js # Command not supported in mongos
  - jstests/core/dropdb_race.js  # syncdelay.
  - jstests/core/fsync.js  # uses fsync.
  - jstests/core/geo_haystack*.js  # geoSearch.
  - jstests/core/geo_s2cursorlimitskip.js  # profiling.
  - jstests/core/geo_update_btree2.js  # notablescan.
  - jstests/core/index_id_options.js  # "local" database.
  - jstests/core/index9.js  # "local" database.
  - jstests/core/queryoptimizera.js  # "local" database.
  - jstests/core/rename*.js # renameCollection.
  - jstests/core/stages*.js  # stageDebug.
  - jstests/core/startup_log.js  # "local" database.
  - jstests/core/tailable_cursor_invalidation.js # capped collections.
  - jstests/core/tailable_getmore_batch_size.js # capped collections.
  - jstests/core/tailable_skip_limit.js # capped collections.
  - jstests/core/top.js  # top.
  # The following tests fail because mongos behaves differently from mongod when testing certain
  # functionality. The differences are in a comment next to the failing test.
  - jstests/core/explain_missing_database.js  # Behavior with no db different on mongos, SERVER-18047.
  - jstests/core/geo_2d_explain.js  # executionSuccess in different spot in explain().
  - jstests/core/geo_s2explain.js  # inputStage in different spot in explain().
  - jstests/core/geo_s2sparse.js  # keysPerIndex in different spot in validate().
  - jstests/core/operation_latency_histogram.js  # Stats are counted differently on mongos, SERVER-24880.
  # Test expects to run against a standalone, but shards in this passthrough are repl sets.
  - jstests/core/read_after_optime.js
  # The following tests fail because they count indexes. These counts do not take into account the
  # additional hashed shard key indexes that are automatically added by this passthrough.
  - jstests/core/apitest_dbcollection.js
  - jstests/core/bad_index_plugin.js
  - jstests/core/create_indexes.js
  - jstests/core/list_indexes_non_existent_ns.js
  - jstests/core/mr_preserve_indexes.js
  - jstests/core/rename6.js
  # The following tests fail because they expect no databases to be created. However a DB is created
  # automatically when we shard a collection.
  - jstests/core/dbcase.js
  - jstests/core/dbcase2.js
  - jstests/core/no_db_created.js
  - jstests/core/views/*.js # Views tests aren't expected to work when collections are implicitly sharded.
  - jstests/core/killop_drop_collection.js # Uses fsyncLock.
  # These tests fail because sharded clusters do not clean up correctly after failed index builds.
  # See SERVER-33207 as an example.
  - jstests/core/geo_borders.js
  # TODO: Remove after fixing SERVER-29449. executionStats.nReturned is incorrect for sharded
  # queries with a limit or for distinct commands.
  - jstests/core/distinct_index1.js
  - jstests/core/explain1.js
  - jstests/core/explain4.js
  - jstests/core/sortk.js
  # TODO: Remove after fixing SERVER-32563. The implementation of explain for the count command is
  # incorrect on sharded collections.
  - jstests/core/explain_count.js
  # TODO SERVER-32311: These tests use plan stage helpers which can't handle sharded explain output.
  - jstests/core/expr_index_use.js
  - jstests/core/index_multikey.js
  - jstests/core/optimized_match_explain.js
  - jstests/core/sort_array.js
  # Create index command arbitrarily picks one of the errors if there are multiple errors from the
  # shards. In this suite, it is possible that one of the shards don't have the collection created
  # so it can return cannot implicitly create collection.
  - jstests/core/index_create_too_many.js
  - jstests/core/index_create_with_nul_in_name.js

  ##
  ## Limitations with the way the runner file injects transactions.
  ##

  # These tests expects some statements to error, which will cause txns to abort entirely.
  - jstests/core/bulk_api_ordered.js
  - jstests/core/bulk_api_unordered.js
  - jstests/core/bulk_legacy_enforce_gle.js
  - jstests/core/capped5.js
  - jstests/core/commands_with_uuid.js
  - jstests/core/dbcase.js
  - jstests/core/dbcase2.js
  - jstests/core/explain_execution_error.js
  - jstests/core/expr.js
  - jstests/core/find9.js
  - jstests/core/find_getmore_bsonsize.js
  - jstests/core/find_getmore_cmd.js
  - jstests/core/geo_allowedcomparisons.js
  - jstests/core/geo_big_polygon2.js
  - jstests/core/geonear_key.js
  - jstests/core/in.js
  - jstests/core/index8.js # No explicit check for failed command.
  - jstests/core/index_decimal.js
  - jstests/core/index_large_and_small_dates.js
  - jstests/core/index_multiple_compatibility.js
  - jstests/core/index_partial_write_ops.js
  - jstests/core/indexa.js # No explicit check for failed command.
  - jstests/core/indexes_multiple_commands.js
  - jstests/core/js2.js
  - jstests/core/json_schema/json_schema.js
  - jstests/core/mr_bigobject.js
  - jstests/core/not2.js
  - jstests/core/null_query_semantics.js
  - jstests/core/or1.js
  - jstests/core/or2.js
  - jstests/core/or3.js
  - jstests/core/orj.js
  - jstests/core/ref.js
  - jstests/core/ref4.js
  - jstests/core/regex_limit.js
  - jstests/core/remove_undefined.js
  - jstests/core/set7.js
  - jstests/core/sortb.js
  - jstests/core/sortf.js
  - jstests/core/sortg.js
  - jstests/core/sortj.js
  - jstests/core/sort_with_meta_operator.js
  - jstests/core/tailable_skip_limit.js
  - jstests/core/type_array.js
  - jstests/core/uniqueness.js
  - jstests/core/unset2.js
  - jstests/core/update_addToSet.js
  - jstests/core/update_array_offset_positional.js
  - jstests/core/update_arrayFilters.js
  - jstests/core/update_find_and_modify_id.js
  - jstests/core/update_modifier_pop.js
  - jstests/core/updateh.js
  - jstests/core/updatej.js

  # SERVER-38096 Mongos opCounter doesn't increase when last op causes the transaction to implicitly abort
  - jstests/core/opcounters_write_cmd.js

  # TODO: SERVER-38207 Cannot insert document with MaxKey shard key.
  - jstests/core/type8.js

  # TODO: SERVER-38686 createIndex should prioritize other errors over
  # CannotImplicitlyCreateCollection
  - jstests/core/fts_spanish.js
  - jstests/core/geo_invalid_polygon.js
  - jstests/core/geo_s2index.js
  - jstests/core/wildcard_index_validindex.js

  # TODO: SERVER-38215 delx.js touches 2 different collections, which makes it susceptible for
  # migration to happen in between and cause the transaction to abort due to StaleConfigException
  # because it is not the first op of the transaction.
  - jstests/core/delx.js

  # benchRun bypasses the network_error_and_txn_override.js so it doesn't run under the same
  # transaction and can cause test to timeout because of the 3 hr transactions lifetime setting and
  # moveChunk getting blocked trying to acquire collection X lock while doing chunk refresh.
  - jstests/core/bench_test*.js
  - jstests/core/benchrun_pipeline_updates.js # benchRun() used for writes

  ##
  ## Some aggregation stages don't support snapshot readconcern.
  ##

  # $explain (requires read concern local)
  - jstests/core/agg_hint.js
  - jstests/core/and.js
  - jstests/core/collation.js
  - jstests/core/explain_shell_helpers.js
  - jstests/core/index_partial_read_ops.js
  - jstests/core/optimized_match_explain.js
  - jstests/core/sort_array.js
  - jstests/core/views/views_collation.js
  - jstests/core/wildcard_index_count.js

  # $listSessions
  - jstests/core/list_all_local_sessions.js
  - jstests/core/list_all_sessions.js
  - jstests/core/list_local_sessions.js
  - jstests/core/list_sessions.js

  # $indexStats
  - jstests/core/index_stats.js

  # $collStats
  - jstests/core/operation_latency_histogram.js
  - jstests/core/views/views_coll_stats.js
  - jstests/core/views/views_stats.js

  # Errors expected to happen in tests, which can cause transactions to get aborted.
  # So when the test tries to inspect the documents it can be out of sync (relative
  # to test run without multi statement transactions).
  - jstests/core/bulk_api_ordered.js
  - jstests/core/bulk_api_unordered.js
  - jstests/core/bulk_legacy_enforce_gle.js
  - jstests/core/cappeda.js
  - jstests/core/doc_validation.js
  - jstests/core/doc_validation_options.js
  - jstests/core/field_name_validation.js
  - jstests/core/insert_illegal_doc.js
  - jstests/core/push_sort.js
  - jstests/core/rename4.js
  - jstests/core/update_arrayFilters.js
  - jstests/core/update_dbref.js
  - jstests/core/updatel.js
  - jstests/core/write_result.js

  # Trick for bypassing mongo shell validation in the test doesn't work because txn_override
  # retry logic will hit the shell validation.
  - jstests/core/invalid_db_name.js

  # Multiple writes in a txn, some of which fail because the collection doesn't exist.
  # We create the collection and retry the last write, but previous writes would have
  # still failed.
  - jstests/core/dbref1.js
  - jstests/core/dbref2.js
  - jstests/core/ref3.js
  - jstests/core/update3.js
  - jstests/core/rename3.js

  ##
  ## Error: Unable to acquire lock within a max lock request timeout of '0ms' milliseconds
  ##

  # Collection drops done through applyOps are not converted to w:majority
  - jstests/core/views/invalid_system_views.js

  ##
  ## Misc. reasons.
  ##

  # SERVER-34868 Cannot run a legacy query on a session.
  - jstests/core/exhaust.js
  - jstests/core/validate_cmd_ns.js

  # SERVER-34772 Tailable Cursors are not allowed with snapshot readconcern.
  - jstests/core/awaitdata_getmore_cmd.js
  - jstests/core/getmore_cmd_maxtimems.js
  - jstests/core/tailable_cursor_invalidation.js
  - jstests/core/tailable_getmore_batch_size.js

  # SERVER-34918 The "max" option of a capped collection can be exceeded until the next insert.
  # The reason is that we don't update the count of a collection until a transaction commits,
  # by which point it is too late to complain that "max" has been exceeded.
  - jstests/core/capped_max1.js

  # The "max" option of a capped collection can be temporarily exceeded before a
  # txn is committed.
  - jstests/core/bulk_insert_capped.js

  # Wrong count for top info (WriteLock)
  - jstests/core/top.js

  # Expects collection to not have been created
  - jstests/core/insert_id_undefined.js

  # Creates sessions explicitly, resulting in txns being run through different sessions
  # using a single txnNumber.
  - jstests/core/json_schema/misc_validation.js
  - jstests/core/views/views_all_commands.js

  # Fails with implicit sessions because it will use multiple sessions on the same Mongo connection.
  - jstests/core/dropdb.js

  # Committing a transaction when the server is fsync locked fails.
  - jstests/core/fsync.js

  # Expects legacy errors ($err).
  - jstests/core/constructors.js

  # txn interrupted by command outside of txn before getMore runs.
  - jstests/core/commands_namespace_parsing.js
  - jstests/core/drop3.js
  - jstests/core/ensure_sorted.js
  - jstests/core/geo_s2cursorlimitskip.js
  - jstests/core/getmore_invalidated_cursors.js
  - jstests/core/getmore_invalidated_documents.js
  - jstests/core/kill_cursors.js
  - jstests/core/list_collections1.js
  - jstests/core/list_indexes.js
  - jstests/core/oro.js

  # Parallel Shell - we do not signal the override to end a txn when a parallel shell closes.
  - jstests/core/awaitdata_getmore_cmd.js
  - jstests/core/compact_keeps_indexes.js
  - jstests/core/count10.js
  - jstests/core/count_plan_summary.js
  - jstests/core/coveredIndex3.js
  - jstests/core/currentop.js
  - jstests/core/distinct3.js
  - jstests/core/find_and_modify_concurrent_update.js
  - jstests/core/fsync.js
  - jstests/core/geo_update_btree.js
  - jstests/core/killop_drop_collection.js
  - jstests/core/loadserverscripts.js
  - jstests/core/mr_killop.js
  - jstests/core/remove9.js
  - jstests/core/removeb.js
  - jstests/core/removec.js
  - jstests/core/shellstartparallel.js
  - jstests/core/updatef.js

  # Command expects to see result from parallel operation.
  # E.g. Suppose the following sequence of events: op1, join() op2 in parallel shell, op3.
  # op3 will still be using the snapshot from op1, and not see op2 at all.
  - jstests/core/bench_test1.js
  - jstests/core/benchrun_pipeline_updates.js
  - jstests/core/cursora.js

  # The tests below use applyOps, SERVER-1439.
  - jstests/core/list_collections1.js
  - jstests/core/list_collections_filter.js
  - jstests/core/rename_stayTemp.js
  - jstests/core/txns/prepare_transaction_fails_on_temp_collections.js

  # Does not support tojson of command objects.
  - jstests/core/SERVER-23626.js

  exclude_with_any_tags:
  # "Cowardly refusing to override read concern of command: ..."
  - assumes_read_concern_unchanged
  # "writeConcern is not allowed within a multi-statement transaction"
  - assumes_write_concern_unchanged
  - assumes_against_mongod_not_mongos
  # Tests tagged with the following will fail because they assume collections are not sharded.
  - assumes_no_implicit_collection_creation_after_drop
  - assumes_no_implicit_index_creation
  - assumes_unsharded_collection
  - cannot_create_unique_index_when_using_hashed_shard_key
  - assumes_balancer_off
  - requires_fastcount
  # Transactions are not allowed to operate on capped collections.
  - requires_profiling
  # Retrying a query can change whether a plan cache entry is active.
  - inspects_whether_plan_cache_entry_is_active
  # $out is not supported in transactions
  - uses_$out

executor:
  archive:
    hooks:
      - CheckReplDBHash
      - ValidateCollections
  config:
    shell_options:
      eval: >-
        var testingReplication = true;
        load('jstests/libs/override_methods/implicitly_shard_accessed_collections.js');
        ImplicitlyShardAccessCollSettings.setMode(ImplicitlyShardAccessCollSettings.Modes.kHashedMoveToSingleShard);
        load('jstests/libs/override_methods/enable_sessions.js');
        load('jstests/libs/override_methods/txn_passthrough_cmd_massage.js');
        load('jstests/libs/override_methods/network_error_and_txn_override.js');
      global_vars:
        TestData:
          networkErrorAndTxnOverrideConfig:
            wrapCRUDinTransactions: true
          sessionOptions:
            # Tests in this suite only read from primaries and only one node is electable, so causal
            # consistency is not required to read your own writes.
            causalConsistency: false
      readMode: commands
  hooks:
  # The CheckReplDBHash hook waits until all operations have replicated to and have been applied
  # on the secondaries, so we run the ValidateCollections hook after it to ensure we're
  # validating the entire contents of the collection.
  - class: CheckReplDBHash
  - class: ValidateCollections
  - class: DropShardedCollections
  - class: CleanEveryN
    n: 20
  fixture:
    class: ShardedClusterFixture
    mongos_options:
      set_parameters:
        enableTestCommands: 1
    mongod_options:
      set_parameters:
        logComponentVerbosity:
          verbosity: 0
          command: 2
          replication:
            election: 4
            heartbeats: 2
            initialSync: 2
            rollback: 2
          storage:
            recovery: 2
          transaction: 4
        enableTestCommands: 1
        numInitialSyncAttempts: 1
        writePeriodicNoops: true
    num_rs_nodes_per_shard: 2
    num_shards: 2