From 22c46665ce1dc0755fb2bbc42ea481c6849aa87c Mon Sep 17 00:00:00 2001 From: Thomas Bloor Date: Mon, 18 Jun 2018 17:21:12 +0100 Subject: [PATCH 1/6] PERL-918 updated retryable-writes test specs --- t/data/retryable-writes/README.rst | 22 +- .../bulkWrite-serverErrors.json | 182 +++ .../bulkWrite-serverErrors.yml | 90 ++ t/data/retryable-writes/bulkWrite.json | 167 ++- t/data/retryable-writes/bulkWrite.yml | 100 +- .../deleteOne-serverErrors.json | 96 ++ .../deleteOne-serverErrors.yml | 50 + t/data/retryable-writes/deleteOne.json | 12 + t/data/retryable-writes/deleteOne.yml | 9 + .../findOneAndDelete-serverErrors.json | 108 ++ .../findOneAndDelete-serverErrors.yml | 50 + t/data/retryable-writes/findOneAndDelete.json | 12 + t/data/retryable-writes/findOneAndDelete.yml | 9 + .../findOneAndReplace-serverErrors.json | 116 ++ .../findOneAndReplace-serverErrors.yml | 54 + .../retryable-writes/findOneAndReplace.json | 12 + t/data/retryable-writes/findOneAndReplace.yml | 9 + .../findOneAndUpdate-serverErrors.json | 118 ++ .../findOneAndUpdate-serverErrors.yml | 54 + t/data/retryable-writes/findOneAndUpdate.json | 12 + t/data/retryable-writes/findOneAndUpdate.yml | 9 + .../insertMany-serverErrors.json | 134 +++ .../insertMany-serverErrors.yml | 59 + t/data/retryable-writes/insertMany.json | 12 + t/data/retryable-writes/insertMany.yml | 9 + .../insertOne-serverErrors.json | 1000 +++++++++++++++++ .../insertOne-serverErrors.yml | 471 ++++++++ t/data/retryable-writes/insertOne.json | 12 + t/data/retryable-writes/insertOne.yml | 9 + .../replaceOne-serverErrors.json | 116 ++ .../replaceOne-serverErrors.yml | 58 + t/data/retryable-writes/replaceOne.json | 12 + t/data/retryable-writes/replaceOne.yml | 9 + .../updateOne-serverErrors.json | 118 ++ .../updateOne-serverErrors.yml | 58 + t/data/retryable-writes/updateOne.json | 24 + t/data/retryable-writes/updateOne.yml | 18 + 37 files changed, 3398 insertions(+), 12 deletions(-) create mode 100644 t/data/retryable-writes/bulkWrite-serverErrors.json create mode 100644 t/data/retryable-writes/bulkWrite-serverErrors.yml create mode 100644 t/data/retryable-writes/deleteOne-serverErrors.json create mode 100644 t/data/retryable-writes/deleteOne-serverErrors.yml create mode 100644 t/data/retryable-writes/findOneAndDelete-serverErrors.json create mode 100644 t/data/retryable-writes/findOneAndDelete-serverErrors.yml create mode 100644 t/data/retryable-writes/findOneAndReplace-serverErrors.json create mode 100644 t/data/retryable-writes/findOneAndReplace-serverErrors.yml create mode 100644 t/data/retryable-writes/findOneAndUpdate-serverErrors.json create mode 100644 t/data/retryable-writes/findOneAndUpdate-serverErrors.yml create mode 100644 t/data/retryable-writes/insertMany-serverErrors.json create mode 100644 t/data/retryable-writes/insertMany-serverErrors.yml create mode 100644 t/data/retryable-writes/insertOne-serverErrors.json create mode 100644 t/data/retryable-writes/insertOne-serverErrors.yml create mode 100644 t/data/retryable-writes/replaceOne-serverErrors.json create mode 100644 t/data/retryable-writes/replaceOne-serverErrors.yml create mode 100644 t/data/retryable-writes/updateOne-serverErrors.json create mode 100644 t/data/retryable-writes/updateOne-serverErrors.yml diff --git a/t/data/retryable-writes/README.rst b/t/data/retryable-writes/README.rst index b9f1008c..606ba93b 100644 --- a/t/data/retryable-writes/README.rst +++ b/t/data/retryable-writes/README.rst @@ -15,10 +15,11 @@ that drivers can use to prove their conformance to the Retryable Writes spec. Several prose tests, which are not easily expressed in YAML, are also presented in this file. Those tests will need to be manually implemented by each driver. -Tests will require a MongoClient with ``retryWrites`` enabled. Integration tests -will require a running MongoDB cluster with server versions 3.6.0 or later. The -``{setFeatureCompatibilityVersion: 3.6}`` admin command will also need to have -been executed to enable support for retryable writes on the cluster. +Tests will require a MongoClient created with options defined in the tests. +Integration tests will require a running MongoDB cluster with server versions +3.6.0 or later. The ``{setFeatureCompatibilityVersion: 3.6}`` admin command +will also need to have been executed to enable support for retryable writes on +the cluster. Server Fail Point ================= @@ -137,15 +138,16 @@ Each YAML file has the following keys: - ``description``: The name of the test. - - ``failPoint``: Document describing options for configuring the - ``onPrimaryTransactionalWrite`` fail point on the primary server. This - document should be merged with the - ``{ configureFailPoint: "onPrimaryTransactionalWrite" }`` command document. + - ``clientOptions``: Parameters to pass to MongoClient(). + + - ``failPoint``: The ``configureFailPoint`` command document to run to + configure a fail point on the primary server. Drivers must ensure that + ``configureFailPoint`` is the first field in the command. - ``operation``: Document describing the operation to be executed. The operation should be executed through a collection object derived from a - client that has been created with the ``retryWrites=true`` option. - This will have some or all of the following fields: + client that has been created with ``clientOptions``. The operation will have + some or all of the following fields: - ``name``: The name of the operation as defined in the CRUD specification. diff --git a/t/data/retryable-writes/bulkWrite-serverErrors.json b/t/data/retryable-writes/bulkWrite-serverErrors.json new file mode 100644 index 00000000..cd81fc61 --- /dev/null +++ b/t/data/retryable-writes/bulkWrite-serverErrors.json @@ -0,0 +1,182 @@ +{ + "data": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + } + ], + "minServerVersion": "3.99", + "tests": [ + { + "description": "BulkWrite succeeds after PrimarySteppedDown", + "clientOptions": { + "retryWrites": true + }, + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "update" + ], + "errorCode": 189 + } + }, + "operation": { + "name": "bulkWrite", + "arguments": { + "requests": [ + { + "name": "deleteOne", + "arguments": { + "filter": { + "_id": 1 + } + } + }, + { + "name": "insertOne", + "arguments": { + "document": { + "_id": 3, + "x": 33 + } + } + }, + { + "name": "updateOne", + "arguments": { + "filter": { + "_id": 2 + }, + "update": { + "$inc": { + "x": 1 + } + } + } + } + ], + "options": { + "ordered": true + } + } + }, + "outcome": { + "result": { + "deletedCount": 1, + "insertedIds": { + "1": 3 + }, + "matchedCount": 1, + "modifiedCount": 1, + "upsertedCount": 0, + "upsertedIds": {} + }, + "collection": { + "data": [ + { + "_id": 2, + "x": 23 + }, + { + "_id": 3, + "x": 33 + } + ] + } + } + }, + { + "description": "BulkWrite succeeds after WriteConcernError ShutdownInProgress", + "clientOptions": { + "retryWrites": true + }, + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "insert" + ], + "writeConcernError": { + "code": 91, + "errmsg": "Replication is being shut down" + } + } + }, + "operation": { + "name": "bulkWrite", + "arguments": { + "requests": [ + { + "name": "deleteOne", + "arguments": { + "filter": { + "_id": 1 + } + } + }, + { + "name": "insertOne", + "arguments": { + "document": { + "_id": 3, + "x": 33 + } + } + }, + { + "name": "updateOne", + "arguments": { + "filter": { + "_id": 2 + }, + "update": { + "$inc": { + "x": 1 + } + } + } + } + ], + "options": { + "ordered": true + } + } + }, + "outcome": { + "result": { + "deletedCount": 1, + "insertedIds": { + "1": 3 + }, + "matchedCount": 1, + "modifiedCount": 1, + "upsertedCount": 0, + "upsertedIds": {} + }, + "collection": { + "data": [ + { + "_id": 2, + "x": 23 + }, + { + "_id": 3, + "x": 33 + } + ] + } + } + } + ] +} diff --git a/t/data/retryable-writes/bulkWrite-serverErrors.yml b/t/data/retryable-writes/bulkWrite-serverErrors.yml new file mode 100644 index 00000000..de9cff5e --- /dev/null +++ b/t/data/retryable-writes/bulkWrite-serverErrors.yml @@ -0,0 +1,90 @@ +data: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + +# TODO: this should change to 4.0 once 4.0.0 is released. +minServerVersion: '3.99' + +tests: + - + description: "BulkWrite succeeds after PrimarySteppedDown" + clientOptions: + retryWrites: true + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["update"] + errorCode: 189 + operation: + name: "bulkWrite" + arguments: + requests: + - + name: "deleteOne" + arguments: + filter: { _id: 1 } + - + name: "insertOne" + arguments: + document: { _id: 3, x: 33 } + - + name: "updateOne" + arguments: + filter: { _id: 2 } + update: { $inc: { x : 1 }} + options: { ordered: true } + outcome: + result: + deletedCount: 1 + insertedIds: { 1: 3 } + matchedCount: 1 + modifiedCount: 1 + upsertedCount: 0 + upsertedIds: { } + collection: + data: + - { _id: 2, x: 23 } + - { _id: 3, x: 33 } + - + description: "BulkWrite succeeds after WriteConcernError ShutdownInProgress" + clientOptions: + retryWrites: true + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["insert"] + writeConcernError: + code: 91 + errmsg: Replication is being shut down + operation: + name: "bulkWrite" + arguments: + requests: + - + name: "deleteOne" + arguments: + filter: { _id: 1 } + - + name: "insertOne" + arguments: + document: { _id: 3, x: 33 } + - + name: "updateOne" + arguments: + filter: { _id: 2 } + update: { $inc: { x : 1 }} + options: { ordered: true } + outcome: + result: + deletedCount: 1 + insertedIds: { 1: 3 } + matchedCount: 1 + modifiedCount: 1 + upsertedCount: 0 + upsertedIds: { } + collection: + data: + - { _id: 2, x: 23 } + - { _id: 3, x: 33 } diff --git a/t/data/retryable-writes/bulkWrite.json b/t/data/retryable-writes/bulkWrite.json index 7b88ffb3..6029aece 100644 --- a/t/data/retryable-writes/bulkWrite.json +++ b/t/data/retryable-writes/bulkWrite.json @@ -9,7 +9,11 @@ "tests": [ { "description": "First command is retried", + "clientOptions": { + "retryWrites": true + }, "failPoint": { + "configureFailPoint": "onPrimaryTransactionalWrite", "mode": { "times": 1 } @@ -77,7 +81,11 @@ }, { "description": "All commands are retried", + "clientOptions": { + "retryWrites": true + }, "failPoint": { + "configureFailPoint": "onPrimaryTransactionalWrite", "mode": { "times": 7 } @@ -206,7 +214,11 @@ }, { "description": "Both commands are retried after their first statement fails", + "clientOptions": { + "retryWrites": true + }, "failPoint": { + "configureFailPoint": "onPrimaryTransactionalWrite", "mode": { "times": 2 } @@ -283,7 +295,11 @@ }, { "description": "Second command is retried after its second statement fails", + "clientOptions": { + "retryWrites": true + }, "failPoint": { + "configureFailPoint": "onPrimaryTransactionalWrite", "mode": { "skip": 2 } @@ -360,7 +376,11 @@ }, { "description": "BulkWrite with unordered execution", + "clientOptions": { + "retryWrites": true + }, "failPoint": { + "configureFailPoint": "onPrimaryTransactionalWrite", "mode": { "times": 1 } @@ -425,7 +445,11 @@ }, { "description": "First insertOne is never committed", + "clientOptions": { + "retryWrites": true + }, "failPoint": { + "configureFailPoint": "onPrimaryTransactionalWrite", "mode": { "times": 2 }, @@ -495,7 +519,11 @@ }, { "description": "Second updateOne is never committed", + "clientOptions": { + "retryWrites": true + }, "failPoint": { + "configureFailPoint": "onPrimaryTransactionalWrite", "mode": { "skip": 1 }, @@ -571,7 +599,11 @@ }, { "description": "Third updateOne is never committed", + "clientOptions": { + "retryWrites": true + }, "failPoint": { + "configureFailPoint": "onPrimaryTransactionalWrite", "mode": { "skip": 2 }, @@ -629,7 +661,140 @@ "result": { "deletedCount": 0, "insertedIds": { - "0": 2 + "1": 2 + }, + "matchedCount": 1, + "modifiedCount": 1, + "upsertedCount": 0, + "upsertedIds": {} + }, + "collection": { + "data": [ + { + "_id": 1, + "x": 12 + }, + { + "_id": 2, + "x": 22 + } + ] + } + } + }, + { + "description": "Single-document write following deleteMany is retried", + "clientOptions": { + "retryWrites": true + }, + "failPoint": { + "configureFailPoint": "onPrimaryTransactionalWrite", + "mode": { + "times": 1 + }, + "data": { + "failBeforeCommitExceptionCode": 1 + } + }, + "operation": { + "name": "bulkWrite", + "arguments": { + "requests": [ + { + "name": "deleteMany", + "arguments": { + "filter": { + "x": 11 + } + } + }, + { + "name": "insertOne", + "arguments": { + "document": { + "_id": 2, + "x": 22 + } + } + } + ], + "options": { + "ordered": true + } + } + }, + "outcome": { + "result": { + "deletedCount": 1, + "insertedIds": { + "1": 2 + }, + "matchedCount": 0, + "modifiedCount": 0, + "upsertedCount": 0, + "upsertedIds": {} + }, + "collection": { + "data": [ + { + "_id": 2, + "x": 22 + } + ] + } + } + }, + { + "description": "Single-document write following updateMany is retried", + "clientOptions": { + "retryWrites": true + }, + "failPoint": { + "configureFailPoint": "onPrimaryTransactionalWrite", + "mode": { + "times": 1 + }, + "data": { + "failBeforeCommitExceptionCode": 1 + } + }, + "operation": { + "name": "bulkWrite", + "arguments": { + "requests": [ + { + "name": "updateMany", + "arguments": { + "filter": { + "x": 11 + }, + "update": { + "$inc": { + "x": 1 + } + } + } + }, + { + "name": "insertOne", + "arguments": { + "document": { + "_id": 2, + "x": 22 + } + } + } + ], + "options": { + "ordered": true + } + } + }, + "outcome": { + "result": { + "deletedCount": 0, + "insertedIds": { + "1": 2 }, "matchedCount": 1, "modifiedCount": 1, diff --git a/t/data/retryable-writes/bulkWrite.yml b/t/data/retryable-writes/bulkWrite.yml index 167f8943..408aba7e 100644 --- a/t/data/retryable-writes/bulkWrite.yml +++ b/t/data/retryable-writes/bulkWrite.yml @@ -6,7 +6,10 @@ minServerVersion: '3.6' tests: - description: "First command is retried" + clientOptions: + retryWrites: true failPoint: + configureFailPoint: onPrimaryTransactionalWrite mode: { times: 1 } operation: name: "bulkWrite" @@ -42,7 +45,10 @@ tests: # that each write command consists of a single statement, which will # fail on the first attempt and succeed on the second, retry attempt. description: "All commands are retried" + clientOptions: + retryWrites: true failPoint: + configureFailPoint: onPrimaryTransactionalWrite mode: { times: 7 } operation: name: "bulkWrite" @@ -97,7 +103,10 @@ tests: - { _id: 5, x: 55 } - description: "Both commands are retried after their first statement fails" + clientOptions: + retryWrites: true failPoint: + configureFailPoint: onPrimaryTransactionalWrite mode: { times: 2 } operation: name: "bulkWrite" @@ -132,7 +141,10 @@ tests: - { _id: 2, x: 23 } - description: "Second command is retried after its second statement fails" + clientOptions: + retryWrites: true failPoint: + configureFailPoint: onPrimaryTransactionalWrite mode: { skip: 2 } operation: name: "bulkWrite" @@ -167,7 +179,10 @@ tests: - { _id: 2, x: 23 } - description: "BulkWrite with unordered execution" + clientOptions: + retryWrites: true failPoint: + configureFailPoint: onPrimaryTransactionalWrite mode: { times: 1 } operation: name: "bulkWrite" @@ -197,7 +212,10 @@ tests: - { _id: 3, x: 33 } - description: "First insertOne is never committed" + clientOptions: + retryWrites: true failPoint: + configureFailPoint: onPrimaryTransactionalWrite mode: { times: 2 } data: { failBeforeCommitExceptionCode: 1 } operation: @@ -232,7 +250,10 @@ tests: - { _id: 1, x: 11 } - description: "Second updateOne is never committed" + clientOptions: + retryWrites: true failPoint: + configureFailPoint: onPrimaryTransactionalWrite mode: { skip: 1 } data: { failBeforeCommitExceptionCode: 1 } operation: @@ -268,7 +289,10 @@ tests: - { _id: 2, x: 22 } - description: "Third updateOne is never committed" + clientOptions: + retryWrites: true failPoint: + configureFailPoint: onPrimaryTransactionalWrite mode: { skip: 2 } data: { failBeforeCommitExceptionCode: 1 } operation: @@ -294,7 +318,81 @@ tests: error: true result: deletedCount: 0 - insertedIds: { 0: 2 } + insertedIds: { 1: 2 } + matchedCount: 1 + modifiedCount: 1 + upsertedCount: 0 + upsertedIds: { } + collection: + data: + - { _id: 1, x: 12 } + - { _id: 2, x: 22 } + - + # The onPrimaryTransactionalWrite fail point only triggers for write + # operations that include a transaction ID. Therefore, it will not + # affect the initial deleteMany and will trigger once (and only once) + # for the first insertOne attempt. + description: "Single-document write following deleteMany is retried" + clientOptions: + retryWrites: true + failPoint: + configureFailPoint: onPrimaryTransactionalWrite + mode: { times: 1 } + data: { failBeforeCommitExceptionCode: 1 } + operation: + name: "bulkWrite" + arguments: + requests: + - + name: "deleteMany" + arguments: + filter: { x: 11 } + - + name: "insertOne" + arguments: + document: { _id: 2, x: 22 } + options: { ordered: true } + outcome: + result: + deletedCount: 1 + insertedIds: { 1: 2 } + matchedCount: 0 + modifiedCount: 0 + upsertedCount: 0 + upsertedIds: { } + collection: + data: + - { _id: 2, x: 22 } + - + # The onPrimaryTransactionalWrite fail point only triggers for write + # operations that include a transaction ID. Therefore, it will not + # affect the initial updateMany and will trigger once (and only once) + # for the first insertOne attempt. + description: "Single-document write following updateMany is retried" + clientOptions: + retryWrites: true + failPoint: + configureFailPoint: onPrimaryTransactionalWrite + mode: { times: 1 } + data: { failBeforeCommitExceptionCode: 1 } + operation: + name: "bulkWrite" + arguments: + requests: + - + name: "updateMany" + arguments: + filter: { x: 11 } + update: { $inc: { x : 1 }} + - + name: "insertOne" + arguments: + document: { _id: 2, x: 22 } + options: { ordered: true } + outcome: + result: + deletedCount: 0 + insertedIds: { 1: 2 } matchedCount: 1 modifiedCount: 1 upsertedCount: 0 diff --git a/t/data/retryable-writes/deleteOne-serverErrors.json b/t/data/retryable-writes/deleteOne-serverErrors.json new file mode 100644 index 00000000..606fc66f --- /dev/null +++ b/t/data/retryable-writes/deleteOne-serverErrors.json @@ -0,0 +1,96 @@ +{ + "data": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + } + ], + "minServerVersion": "3.99", + "tests": [ + { + "description": "DeleteOne succeeds after PrimarySteppedDown", + "clientOptions": { + "retryWrites": true + }, + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "delete" + ], + "errorCode": 189 + } + }, + "operation": { + "name": "deleteOne", + "arguments": { + "filter": { + "_id": 1 + } + } + }, + "outcome": { + "result": { + "deletedCount": 1 + }, + "collection": { + "data": [ + { + "_id": 2, + "x": 22 + } + ] + } + } + }, + { + "description": "DeleteOne succeeds after WriteConcernError ShutdownInProgress", + "clientOptions": { + "retryWrites": true + }, + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "delete" + ], + "writeConcernError": { + "code": 91, + "errmsg": "Replication is being shut down" + } + } + }, + "operation": { + "name": "deleteOne", + "arguments": { + "filter": { + "_id": 1 + } + } + }, + "outcome": { + "result": { + "deletedCount": 1 + }, + "collection": { + "data": [ + { + "_id": 2, + "x": 22 + } + ] + } + } + } + ] +} diff --git a/t/data/retryable-writes/deleteOne-serverErrors.yml b/t/data/retryable-writes/deleteOne-serverErrors.yml new file mode 100644 index 00000000..006f1513 --- /dev/null +++ b/t/data/retryable-writes/deleteOne-serverErrors.yml @@ -0,0 +1,50 @@ +data: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + +# TODO: this should change to 4.0 once 4.0.0 is released. +minServerVersion: '3.99' + +tests: + - + description: "DeleteOne succeeds after PrimarySteppedDown" + clientOptions: + retryWrites: true + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["delete"] + errorCode: 189 + operation: + name: "deleteOne" + arguments: + filter: { _id: 1 } + outcome: + result: + deletedCount: 1 + collection: + data: + - { _id: 2, x: 22 } + - + description: "DeleteOne succeeds after WriteConcernError ShutdownInProgress" + clientOptions: + retryWrites: true + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["delete"] + writeConcernError: + code: 91 + errmsg: Replication is being shut down + operation: + name: "deleteOne" + arguments: + filter: { _id: 1 } + outcome: + result: + deletedCount: 1 + collection: + data: + - { _id: 2, x: 22 } diff --git a/t/data/retryable-writes/deleteOne.json b/t/data/retryable-writes/deleteOne.json index a552b893..8e42e9c4 100644 --- a/t/data/retryable-writes/deleteOne.json +++ b/t/data/retryable-writes/deleteOne.json @@ -13,7 +13,11 @@ "tests": [ { "description": "DeleteOne is committed on first attempt", + "clientOptions": { + "retryWrites": true + }, "failPoint": { + "configureFailPoint": "onPrimaryTransactionalWrite", "mode": { "times": 1 } @@ -42,7 +46,11 @@ }, { "description": "DeleteOne is not committed on first attempt", + "clientOptions": { + "retryWrites": true + }, "failPoint": { + "configureFailPoint": "onPrimaryTransactionalWrite", "mode": { "times": 1 }, @@ -74,7 +82,11 @@ }, { "description": "DeleteOne is never committed", + "clientOptions": { + "retryWrites": true + }, "failPoint": { + "configureFailPoint": "onPrimaryTransactionalWrite", "mode": { "times": 2 }, diff --git a/t/data/retryable-writes/deleteOne.yml b/t/data/retryable-writes/deleteOne.yml index 20d61966..d82bfc05 100644 --- a/t/data/retryable-writes/deleteOne.yml +++ b/t/data/retryable-writes/deleteOne.yml @@ -7,7 +7,10 @@ minServerVersion: '3.6' tests: - description: "DeleteOne is committed on first attempt" + clientOptions: + retryWrites: true failPoint: + configureFailPoint: onPrimaryTransactionalWrite mode: { times: 1 } operation: name: "deleteOne" @@ -21,7 +24,10 @@ tests: - { _id: 2, x: 22 } - description: "DeleteOne is not committed on first attempt" + clientOptions: + retryWrites: true failPoint: + configureFailPoint: onPrimaryTransactionalWrite mode: { times: 1 } data: { failBeforeCommitExceptionCode: 1 } operation: @@ -36,7 +42,10 @@ tests: - { _id: 2, x: 22 } - description: "DeleteOne is never committed" + clientOptions: + retryWrites: true failPoint: + configureFailPoint: onPrimaryTransactionalWrite mode: { times: 2 } data: { failBeforeCommitExceptionCode: 1 } operation: diff --git a/t/data/retryable-writes/findOneAndDelete-serverErrors.json b/t/data/retryable-writes/findOneAndDelete-serverErrors.json new file mode 100644 index 00000000..306b1423 --- /dev/null +++ b/t/data/retryable-writes/findOneAndDelete-serverErrors.json @@ -0,0 +1,108 @@ +{ + "data": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + } + ], + "minServerVersion": "3.99", + "tests": [ + { + "description": "FindOneAndDelete succeeds after PrimarySteppedDown", + "clientOptions": { + "retryWrites": true + }, + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "findAndModify" + ], + "errorCode": 189 + } + }, + "operation": { + "name": "findOneAndDelete", + "arguments": { + "filter": { + "x": { + "$gte": 11 + } + }, + "sort": { + "x": 1 + } + } + }, + "outcome": { + "result": { + "_id": 1, + "x": 11 + }, + "collection": { + "data": [ + { + "_id": 2, + "x": 22 + } + ] + } + } + }, + { + "description": "FindOneAndDelete succeeds after WriteConcernError ShutdownInProgress", + "clientOptions": { + "retryWrites": true + }, + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "findAndModify" + ], + "writeConcernError": { + "code": 91, + "errmsg": "Replication is being shut down" + } + } + }, + "operation": { + "name": "findOneAndDelete", + "arguments": { + "filter": { + "x": { + "$gte": 11 + } + }, + "sort": { + "x": 1 + } + } + }, + "outcome": { + "result": { + "_id": 1, + "x": 11 + }, + "collection": { + "data": [ + { + "_id": 2, + "x": 22 + } + ] + } + } + } + ] +} diff --git a/t/data/retryable-writes/findOneAndDelete-serverErrors.yml b/t/data/retryable-writes/findOneAndDelete-serverErrors.yml new file mode 100644 index 00000000..75071a36 --- /dev/null +++ b/t/data/retryable-writes/findOneAndDelete-serverErrors.yml @@ -0,0 +1,50 @@ +data: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + +# TODO: this should change to 4.0 once 4.0.0 is released. +minServerVersion: '3.99' + +tests: + - + description: "FindOneAndDelete succeeds after PrimarySteppedDown" + clientOptions: + retryWrites: true + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["findAndModify"] + errorCode: 189 + operation: + name: "findOneAndDelete" + arguments: + filter: { x: { $gte: 11 }} + sort: { x: 1 } + outcome: + result: { _id: 1, x: 11 } + collection: + data: + - { _id: 2, x: 22 } + - + description: "FindOneAndDelete succeeds after WriteConcernError ShutdownInProgress" + clientOptions: + retryWrites: true + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["findAndModify"] + writeConcernError: + code: 91 + errmsg: Replication is being shut down + operation: + name: "findOneAndDelete" + arguments: + filter: { x: { $gte: 11 }} + sort: { x: 1 } + outcome: + result: { _id: 1, x: 11 } + collection: + data: + - { _id: 2, x: 22 } diff --git a/t/data/retryable-writes/findOneAndDelete.json b/t/data/retryable-writes/findOneAndDelete.json index d8f6c8fa..5963c432 100644 --- a/t/data/retryable-writes/findOneAndDelete.json +++ b/t/data/retryable-writes/findOneAndDelete.json @@ -13,7 +13,11 @@ "tests": [ { "description": "FindOneAndDelete is committed on first attempt", + "clientOptions": { + "retryWrites": true + }, "failPoint": { + "configureFailPoint": "onPrimaryTransactionalWrite", "mode": { "times": 1 } @@ -48,7 +52,11 @@ }, { "description": "FindOneAndDelete is not committed on first attempt", + "clientOptions": { + "retryWrites": true + }, "failPoint": { + "configureFailPoint": "onPrimaryTransactionalWrite", "mode": { "times": 1 }, @@ -86,7 +94,11 @@ }, { "description": "FindOneAndDelete is never committed", + "clientOptions": { + "retryWrites": true + }, "failPoint": { + "configureFailPoint": "onPrimaryTransactionalWrite", "mode": { "times": 2 }, diff --git a/t/data/retryable-writes/findOneAndDelete.yml b/t/data/retryable-writes/findOneAndDelete.yml index d945bc4d..4f5d71ca 100644 --- a/t/data/retryable-writes/findOneAndDelete.yml +++ b/t/data/retryable-writes/findOneAndDelete.yml @@ -7,7 +7,10 @@ minServerVersion: '3.6' tests: - description: "FindOneAndDelete is committed on first attempt" + clientOptions: + retryWrites: true failPoint: + configureFailPoint: onPrimaryTransactionalWrite mode: { times: 1 } operation: name: "findOneAndDelete" @@ -21,7 +24,10 @@ tests: - { _id: 2, x: 22 } - description: "FindOneAndDelete is not committed on first attempt" + clientOptions: + retryWrites: true failPoint: + configureFailPoint: onPrimaryTransactionalWrite mode: { times: 1 } data: { failBeforeCommitExceptionCode: 1 } operation: @@ -36,7 +42,10 @@ tests: - { _id: 2, x: 22 } - description: "FindOneAndDelete is never committed" + clientOptions: + retryWrites: true failPoint: + configureFailPoint: onPrimaryTransactionalWrite mode: { times: 2 } data: { failBeforeCommitExceptionCode: 1 } operation: diff --git a/t/data/retryable-writes/findOneAndReplace-serverErrors.json b/t/data/retryable-writes/findOneAndReplace-serverErrors.json new file mode 100644 index 00000000..ab209b8d --- /dev/null +++ b/t/data/retryable-writes/findOneAndReplace-serverErrors.json @@ -0,0 +1,116 @@ +{ + "data": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + } + ], + "minServerVersion": "3.99", + "tests": [ + { + "description": "FindOneAndReplace succeeds after PrimarySteppedDown", + "clientOptions": { + "retryWrites": true + }, + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "findAndModify" + ], + "errorCode": 189 + } + }, + "operation": { + "name": "findOneAndReplace", + "arguments": { + "filter": { + "_id": 1 + }, + "replacement": { + "_id": 1, + "x": 111 + }, + "returnDocument": "Before" + } + }, + "outcome": { + "result": { + "_id": 1, + "x": 11 + }, + "collection": { + "data": [ + { + "_id": 1, + "x": 111 + }, + { + "_id": 2, + "x": 22 + } + ] + } + } + }, + { + "description": "FindOneAndReplace succeeds after WriteConcernError ShutdownInProgress", + "clientOptions": { + "retryWrites": true + }, + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "findAndModify" + ], + "writeConcernError": { + "code": 91, + "errmsg": "Replication is being shut down" + } + } + }, + "operation": { + "name": "findOneAndReplace", + "arguments": { + "filter": { + "_id": 1 + }, + "replacement": { + "_id": 1, + "x": 111 + }, + "returnDocument": "Before" + } + }, + "outcome": { + "result": { + "_id": 1, + "x": 11 + }, + "collection": { + "data": [ + { + "_id": 1, + "x": 111 + }, + { + "_id": 2, + "x": 22 + } + ] + } + } + } + ] +} diff --git a/t/data/retryable-writes/findOneAndReplace-serverErrors.yml b/t/data/retryable-writes/findOneAndReplace-serverErrors.yml new file mode 100644 index 00000000..8b6a2012 --- /dev/null +++ b/t/data/retryable-writes/findOneAndReplace-serverErrors.yml @@ -0,0 +1,54 @@ +data: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + +# TODO: this should change to 4.0 once 4.0.0 is released. +minServerVersion: '3.99' + +tests: + - + description: "FindOneAndReplace succeeds after PrimarySteppedDown" + clientOptions: + retryWrites: true + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["findAndModify"] + errorCode: 189 + operation: + name: "findOneAndReplace" + arguments: + filter: { _id: 1 } + replacement: { _id: 1, x: 111 } + returnDocument: "Before" + outcome: + result: { _id: 1, x: 11 } + collection: + data: + - { _id: 1, x: 111 } + - { _id: 2, x: 22 } + - + description: "FindOneAndReplace succeeds after WriteConcernError ShutdownInProgress" + clientOptions: + retryWrites: true + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["findAndModify"] + writeConcernError: + code: 91 + errmsg: Replication is being shut down + operation: + name: "findOneAndReplace" + arguments: + filter: { _id: 1 } + replacement: { _id: 1, x: 111 } + returnDocument: "Before" + outcome: + result: { _id: 1, x: 11 } + collection: + data: + - { _id: 1, x: 111 } + - { _id: 2, x: 22 } diff --git a/t/data/retryable-writes/findOneAndReplace.json b/t/data/retryable-writes/findOneAndReplace.json index 22c03d74..20b9e0bc 100644 --- a/t/data/retryable-writes/findOneAndReplace.json +++ b/t/data/retryable-writes/findOneAndReplace.json @@ -13,7 +13,11 @@ "tests": [ { "description": "FindOneAndReplace is committed on first attempt", + "clientOptions": { + "retryWrites": true + }, "failPoint": { + "configureFailPoint": "onPrimaryTransactionalWrite", "mode": { "times": 1 } @@ -52,7 +56,11 @@ }, { "description": "FindOneAndReplace is not committed on first attempt", + "clientOptions": { + "retryWrites": true + }, "failPoint": { + "configureFailPoint": "onPrimaryTransactionalWrite", "mode": { "times": 1 }, @@ -94,7 +102,11 @@ }, { "description": "FindOneAndReplace is never committed", + "clientOptions": { + "retryWrites": true + }, "failPoint": { + "configureFailPoint": "onPrimaryTransactionalWrite", "mode": { "times": 2 }, diff --git a/t/data/retryable-writes/findOneAndReplace.yml b/t/data/retryable-writes/findOneAndReplace.yml index f2dfc572..6e26d7cf 100644 --- a/t/data/retryable-writes/findOneAndReplace.yml +++ b/t/data/retryable-writes/findOneAndReplace.yml @@ -7,7 +7,10 @@ minServerVersion: '3.6' tests: - description: "FindOneAndReplace is committed on first attempt" + clientOptions: + retryWrites: true failPoint: + configureFailPoint: onPrimaryTransactionalWrite mode: { times: 1 } operation: name: "findOneAndReplace" @@ -23,7 +26,10 @@ tests: - { _id: 2, x: 22 } - description: "FindOneAndReplace is not committed on first attempt" + clientOptions: + retryWrites: true failPoint: + configureFailPoint: onPrimaryTransactionalWrite mode: { times: 1 } data: { failBeforeCommitExceptionCode: 1 } operation: @@ -40,7 +46,10 @@ tests: - { _id: 2, x: 22 } - description: "FindOneAndReplace is never committed" + clientOptions: + retryWrites: true failPoint: + configureFailPoint: onPrimaryTransactionalWrite mode: { times: 2 } data: { failBeforeCommitExceptionCode: 1 } operation: diff --git a/t/data/retryable-writes/findOneAndUpdate-serverErrors.json b/t/data/retryable-writes/findOneAndUpdate-serverErrors.json new file mode 100644 index 00000000..92f09ce9 --- /dev/null +++ b/t/data/retryable-writes/findOneAndUpdate-serverErrors.json @@ -0,0 +1,118 @@ +{ + "data": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + } + ], + "minServerVersion": "3.99", + "tests": [ + { + "description": "FindOneAndUpdate succeeds after PrimarySteppedDown", + "clientOptions": { + "retryWrites": true + }, + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "findAndModify" + ], + "errorCode": 189 + } + }, + "operation": { + "name": "findOneAndUpdate", + "arguments": { + "filter": { + "_id": 1 + }, + "update": { + "$inc": { + "x": 1 + } + }, + "returnDocument": "Before" + } + }, + "outcome": { + "result": { + "_id": 1, + "x": 11 + }, + "collection": { + "data": [ + { + "_id": 1, + "x": 12 + }, + { + "_id": 2, + "x": 22 + } + ] + } + } + }, + { + "description": "FindOneAndUpdate succeeds after WriteConcernError ShutdownInProgress", + "clientOptions": { + "retryWrites": true + }, + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "findAndModify" + ], + "writeConcernError": { + "code": 91, + "errmsg": "Replication is being shut down" + } + } + }, + "operation": { + "name": "findOneAndUpdate", + "arguments": { + "filter": { + "_id": 1 + }, + "update": { + "$inc": { + "x": 1 + } + }, + "returnDocument": "Before" + } + }, + "outcome": { + "result": { + "_id": 1, + "x": 11 + }, + "collection": { + "data": [ + { + "_id": 1, + "x": 12 + }, + { + "_id": 2, + "x": 22 + } + ] + } + } + } + ] +} diff --git a/t/data/retryable-writes/findOneAndUpdate-serverErrors.yml b/t/data/retryable-writes/findOneAndUpdate-serverErrors.yml new file mode 100644 index 00000000..7eddcdc0 --- /dev/null +++ b/t/data/retryable-writes/findOneAndUpdate-serverErrors.yml @@ -0,0 +1,54 @@ +data: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + +# TODO: this should change to 4.0 once 4.0.0 is released. +minServerVersion: '3.99' + +tests: + - + description: "FindOneAndUpdate succeeds after PrimarySteppedDown" + clientOptions: + retryWrites: true + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["findAndModify"] + errorCode: 189 + operation: + name: "findOneAndUpdate" + arguments: + filter: { _id: 1 } + update: { $inc: { x : 1 }} + returnDocument: "Before" + outcome: + result: { _id: 1, x: 11 } + collection: + data: + - { _id: 1, x: 12 } + - { _id: 2, x: 22 } + - + description: "FindOneAndUpdate succeeds after WriteConcernError ShutdownInProgress" + clientOptions: + retryWrites: true + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["findAndModify"] + writeConcernError: + code: 91 + errmsg: Replication is being shut down + operation: + name: "findOneAndUpdate" + arguments: + filter: { _id: 1 } + update: { $inc: { x : 1 }} + returnDocument: "Before" + outcome: + result: { _id: 1, x: 11 } + collection: + data: + - { _id: 1, x: 12 } + - { _id: 2, x: 22 } diff --git a/t/data/retryable-writes/findOneAndUpdate.json b/t/data/retryable-writes/findOneAndUpdate.json index 11a76ab1..92d4f54b 100644 --- a/t/data/retryable-writes/findOneAndUpdate.json +++ b/t/data/retryable-writes/findOneAndUpdate.json @@ -13,7 +13,11 @@ "tests": [ { "description": "FindOneAndUpdate is committed on first attempt", + "clientOptions": { + "retryWrites": true + }, "failPoint": { + "configureFailPoint": "onPrimaryTransactionalWrite", "mode": { "times": 1 } @@ -53,7 +57,11 @@ }, { "description": "FindOneAndUpdate is not committed on first attempt", + "clientOptions": { + "retryWrites": true + }, "failPoint": { + "configureFailPoint": "onPrimaryTransactionalWrite", "mode": { "times": 1 }, @@ -96,7 +104,11 @@ }, { "description": "FindOneAndUpdate is never committed", + "clientOptions": { + "retryWrites": true + }, "failPoint": { + "configureFailPoint": "onPrimaryTransactionalWrite", "mode": { "times": 2 }, diff --git a/t/data/retryable-writes/findOneAndUpdate.yml b/t/data/retryable-writes/findOneAndUpdate.yml index c9fc7672..a82bd034 100644 --- a/t/data/retryable-writes/findOneAndUpdate.yml +++ b/t/data/retryable-writes/findOneAndUpdate.yml @@ -7,7 +7,10 @@ minServerVersion: '3.6' tests: - description: "FindOneAndUpdate is committed on first attempt" + clientOptions: + retryWrites: true failPoint: + configureFailPoint: onPrimaryTransactionalWrite mode: { times: 1 } operation: name: "findOneAndUpdate" @@ -23,7 +26,10 @@ tests: - { _id: 2, x: 22 } - description: "FindOneAndUpdate is not committed on first attempt" + clientOptions: + retryWrites: true failPoint: + configureFailPoint: onPrimaryTransactionalWrite mode: { times: 1 } data: { failBeforeCommitExceptionCode: 1 } operation: @@ -40,7 +46,10 @@ tests: - { _id: 2, x: 22 } - description: "FindOneAndUpdate is never committed" + clientOptions: + retryWrites: true failPoint: + configureFailPoint: onPrimaryTransactionalWrite mode: { times: 2 } data: { failBeforeCommitExceptionCode: 1 } operation: diff --git a/t/data/retryable-writes/insertMany-serverErrors.json b/t/data/retryable-writes/insertMany-serverErrors.json new file mode 100644 index 00000000..17ba9b14 --- /dev/null +++ b/t/data/retryable-writes/insertMany-serverErrors.json @@ -0,0 +1,134 @@ +{ + "data": [ + { + "_id": 1, + "x": 11 + } + ], + "minServerVersion": "3.99", + "tests": [ + { + "description": "InsertMany succeeds after PrimarySteppedDown", + "clientOptions": { + "retryWrites": true + }, + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorCode": 189 + } + }, + "operation": { + "name": "insertMany", + "arguments": { + "documents": [ + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ], + "options": { + "ordered": true + } + } + }, + "outcome": { + "result": { + "insertedIds": { + "0": 2, + "1": 3 + } + }, + "collection": { + "data": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + } + }, + { + "description": "InsertMany succeeds after WriteConcernError ShutdownInProgress", + "clientOptions": { + "retryWrites": true + }, + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "insert" + ], + "writeConcernError": { + "code": 91, + "errmsg": "Replication is being shut down" + } + } + }, + "operation": { + "name": "insertMany", + "arguments": { + "documents": [ + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ], + "options": { + "ordered": true + } + } + }, + "outcome": { + "result": { + "insertedIds": { + "0": 2, + "1": 3 + } + }, + "collection": { + "data": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + } + } + ] +} diff --git a/t/data/retryable-writes/insertMany-serverErrors.yml b/t/data/retryable-writes/insertMany-serverErrors.yml new file mode 100644 index 00000000..aafaee81 --- /dev/null +++ b/t/data/retryable-writes/insertMany-serverErrors.yml @@ -0,0 +1,59 @@ +data: + - { _id: 1, x: 11 } + +# TODO: this should change to 4.0 once 4.0.0 is released. +minServerVersion: '3.99' + +tests: + - + description: "InsertMany succeeds after PrimarySteppedDown" + clientOptions: + retryWrites: true + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["insert"] + errorCode: 189 + operation: + name: "insertMany" + arguments: + documents: + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } + options: { ordered: true } + outcome: + result: + insertedIds: { 0: 2, 1: 3 } + collection: + data: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } + - + description: "InsertMany succeeds after WriteConcernError ShutdownInProgress" + clientOptions: + retryWrites: true + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["insert"] + writeConcernError: + code: 91 + errmsg: Replication is being shut down + operation: + name: "insertMany" + arguments: + documents: + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } + options: { ordered: true } + outcome: + result: + insertedIds: { 0: 2, 1: 3 } + collection: + data: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } diff --git a/t/data/retryable-writes/insertMany.json b/t/data/retryable-writes/insertMany.json index 2d71cb91..74dd4a7a 100644 --- a/t/data/retryable-writes/insertMany.json +++ b/t/data/retryable-writes/insertMany.json @@ -9,7 +9,11 @@ "tests": [ { "description": "InsertMany succeeds after one network error", + "clientOptions": { + "retryWrites": true + }, "failPoint": { + "configureFailPoint": "onPrimaryTransactionalWrite", "mode": { "times": 1 } @@ -59,7 +63,11 @@ }, { "description": "InsertMany with unordered execution", + "clientOptions": { + "retryWrites": true + }, "failPoint": { + "configureFailPoint": "onPrimaryTransactionalWrite", "mode": { "times": 1 } @@ -109,7 +117,11 @@ }, { "description": "InsertMany fails after multiple network errors", + "clientOptions": { + "retryWrites": true + }, "failPoint": { + "configureFailPoint": "onPrimaryTransactionalWrite", "mode": "alwaysOn", "data": { "failBeforeCommitExceptionCode": 1 diff --git a/t/data/retryable-writes/insertMany.yml b/t/data/retryable-writes/insertMany.yml index f3ff5d2f..7559e340 100644 --- a/t/data/retryable-writes/insertMany.yml +++ b/t/data/retryable-writes/insertMany.yml @@ -6,7 +6,10 @@ minServerVersion: '3.6' tests: - description: "InsertMany succeeds after one network error" + clientOptions: + retryWrites: true failPoint: + configureFailPoint: onPrimaryTransactionalWrite mode: { times: 1 } operation: name: "insertMany" @@ -25,7 +28,10 @@ tests: - { _id: 3, x: 33 } - description: "InsertMany with unordered execution" + clientOptions: + retryWrites: true failPoint: + configureFailPoint: onPrimaryTransactionalWrite mode: { times: 1 } operation: name: "insertMany" @@ -44,6 +50,8 @@ tests: - { _id: 3, x: 33 } - description: "InsertMany fails after multiple network errors" + clientOptions: + retryWrites: true failPoint: # Normally, a mongod will insert the documents as a batch with a # single commit. If this fails, mongod may try to insert each @@ -51,6 +59,7 @@ tests: # single insert command may trigger the failpoint twice on each # driver attempt. This test permanently enables the fail point to # ensure the retry attempt always fails. + configureFailPoint: onPrimaryTransactionalWrite mode: "alwaysOn" data: { failBeforeCommitExceptionCode: 1 } operation: diff --git a/t/data/retryable-writes/insertOne-serverErrors.json b/t/data/retryable-writes/insertOne-serverErrors.json new file mode 100644 index 00000000..9ef84370 --- /dev/null +++ b/t/data/retryable-writes/insertOne-serverErrors.json @@ -0,0 +1,1000 @@ +{ + "data": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + } + ], + "minServerVersion": "3.99", + "tests": [ + { + "description": "InsertOne succeeds after connection failure", + "clientOptions": { + "retryWrites": true + }, + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "insert" + ], + "closeConnection": true + } + }, + "operation": { + "name": "insertOne", + "arguments": { + "document": { + "_id": 3, + "x": 33 + } + } + }, + "outcome": { + "result": { + "insertedId": 3 + }, + "collection": { + "data": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + } + }, + { + "description": "InsertOne succeeds after NotMaster", + "clientOptions": { + "retryWrites": true + }, + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorCode": 10107, + "closeConnection": false + } + }, + "operation": { + "name": "insertOne", + "arguments": { + "document": { + "_id": 3, + "x": 33 + } + } + }, + "outcome": { + "result": { + "insertedId": 3 + }, + "collection": { + "data": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + } + }, + { + "description": "InsertOne succeeds after NotMasterOrSecondary", + "clientOptions": { + "retryWrites": true + }, + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorCode": 13436, + "closeConnection": false + } + }, + "operation": { + "name": "insertOne", + "arguments": { + "document": { + "_id": 3, + "x": 33 + } + } + }, + "outcome": { + "result": { + "insertedId": 3 + }, + "collection": { + "data": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + } + }, + { + "description": "InsertOne succeeds after NotMasterNoSlaveOk", + "clientOptions": { + "retryWrites": true + }, + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorCode": 13435, + "closeConnection": false + } + }, + "operation": { + "name": "insertOne", + "arguments": { + "document": { + "_id": 3, + "x": 33 + } + } + }, + "outcome": { + "result": { + "insertedId": 3 + }, + "collection": { + "data": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + } + }, + { + "description": "InsertOne succeeds after InterruptedDueToReplStateChange", + "clientOptions": { + "retryWrites": true + }, + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorCode": 11602, + "closeConnection": false + } + }, + "operation": { + "name": "insertOne", + "arguments": { + "document": { + "_id": 3, + "x": 33 + } + } + }, + "outcome": { + "result": { + "insertedId": 3 + }, + "collection": { + "data": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + } + }, + { + "description": "InsertOne succeeds after InterruptedAtShutdown", + "clientOptions": { + "retryWrites": true + }, + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorCode": 11600, + "closeConnection": false + } + }, + "operation": { + "name": "insertOne", + "arguments": { + "document": { + "_id": 3, + "x": 33 + } + } + }, + "outcome": { + "result": { + "insertedId": 3 + }, + "collection": { + "data": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + } + }, + { + "description": "InsertOne succeeds after PrimarySteppedDown", + "clientOptions": { + "retryWrites": true + }, + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorCode": 189, + "closeConnection": false + } + }, + "operation": { + "name": "insertOne", + "arguments": { + "document": { + "_id": 3, + "x": 33 + } + } + }, + "outcome": { + "result": { + "insertedId": 3 + }, + "collection": { + "data": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + } + }, + { + "description": "InsertOne succeeds after ShutdownInProgress", + "clientOptions": { + "retryWrites": true + }, + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorCode": 91, + "closeConnection": false + } + }, + "operation": { + "name": "insertOne", + "arguments": { + "document": { + "_id": 3, + "x": 33 + } + } + }, + "outcome": { + "result": { + "insertedId": 3 + }, + "collection": { + "data": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + } + }, + { + "description": "InsertOne succeeds after HostNotFound", + "clientOptions": { + "retryWrites": true + }, + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorCode": 7, + "closeConnection": false + } + }, + "operation": { + "name": "insertOne", + "arguments": { + "document": { + "_id": 3, + "x": 33 + } + } + }, + "outcome": { + "result": { + "insertedId": 3 + }, + "collection": { + "data": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + } + }, + { + "description": "InsertOne succeeds after HostUnreachable", + "clientOptions": { + "retryWrites": true + }, + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorCode": 6, + "closeConnection": false + } + }, + "operation": { + "name": "insertOne", + "arguments": { + "document": { + "_id": 3, + "x": 33 + } + } + }, + "outcome": { + "result": { + "insertedId": 3 + }, + "collection": { + "data": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + } + }, + { + "description": "InsertOne succeeds after SocketException", + "clientOptions": { + "retryWrites": true + }, + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorCode": 9001, + "closeConnection": false + } + }, + "operation": { + "name": "insertOne", + "arguments": { + "document": { + "_id": 3, + "x": 33 + } + } + }, + "outcome": { + "result": { + "insertedId": 3 + }, + "collection": { + "data": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + } + }, + { + "description": "InsertOne succeeds after NetworkTimeout", + "clientOptions": { + "retryWrites": true + }, + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorCode": 89, + "closeConnection": false + } + }, + "operation": { + "name": "insertOne", + "arguments": { + "document": { + "_id": 3, + "x": 33 + } + } + }, + "outcome": { + "result": { + "insertedId": 3 + }, + "collection": { + "data": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + } + }, + { + "description": "InsertOne fails after Interrupted", + "clientOptions": { + "retryWrites": true + }, + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorCode": 11601, + "closeConnection": false + } + }, + "operation": { + "name": "insertOne", + "arguments": { + "document": { + "_id": 3, + "x": 33 + } + } + }, + "outcome": { + "error": true, + "collection": { + "data": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + } + ] + } + } + }, + { + "description": "InsertOne succeeds after WriteConcernError InterruptedAtShutdown", + "clientOptions": { + "retryWrites": true + }, + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "insert" + ], + "writeConcernError": { + "code": 11600, + "errmsg": "Replication is being shut down" + } + } + }, + "operation": { + "name": "insertOne", + "arguments": { + "document": { + "_id": 3, + "x": 33 + } + } + }, + "outcome": { + "result": { + "insertedId": 3 + }, + "collection": { + "data": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + } + }, + { + "description": "InsertOne succeeds after WriteConcernError InterruptedDueToReplStateChange", + "clientOptions": { + "retryWrites": true + }, + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "insert" + ], + "writeConcernError": { + "code": 11602, + "errmsg": "Replication is being shut down" + } + } + }, + "operation": { + "name": "insertOne", + "arguments": { + "document": { + "_id": 3, + "x": 33 + } + } + }, + "outcome": { + "result": { + "insertedId": 3 + }, + "collection": { + "data": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + } + }, + { + "description": "InsertOne succeeds after WriteConcernError PrimarySteppedDown", + "clientOptions": { + "retryWrites": true + }, + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "insert" + ], + "writeConcernError": { + "code": 189, + "errmsg": "Replication is being shut down" + } + } + }, + "operation": { + "name": "insertOne", + "arguments": { + "document": { + "_id": 3, + "x": 33 + } + } + }, + "outcome": { + "result": { + "insertedId": 3 + }, + "collection": { + "data": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + } + }, + { + "description": "InsertOne succeeds after WriteConcernError ShutdownInProgress", + "clientOptions": { + "retryWrites": true + }, + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "insert" + ], + "writeConcernError": { + "code": 91, + "errmsg": "Replication is being shut down" + } + } + }, + "operation": { + "name": "insertOne", + "arguments": { + "document": { + "_id": 3, + "x": 33 + } + } + }, + "outcome": { + "result": { + "insertedId": 3 + }, + "collection": { + "data": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + } + }, + { + "description": "InsertOne fails after multiple retryable writeConcernErrors", + "clientOptions": { + "retryWrites": true + }, + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 2 + }, + "data": { + "failCommands": [ + "insert" + ], + "writeConcernError": { + "code": 91, + "errmsg": "Replication is being shut down" + } + } + }, + "operation": { + "name": "insertOne", + "arguments": { + "document": { + "_id": 3, + "x": 33 + } + } + }, + "outcome": { + "error": true, + "collection": { + "data": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + } + }, + { + "description": "InsertOne fails after WriteConcernError Interrupted", + "clientOptions": { + "retryWrites": true + }, + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "insert" + ], + "writeConcernError": { + "code": 11601, + "errmsg": "operation was interrupted" + } + } + }, + "operation": { + "name": "insertOne", + "arguments": { + "document": { + "_id": 3, + "x": 33 + } + } + }, + "outcome": { + "error": true, + "collection": { + "data": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + } + }, + { + "description": "InsertOne fails after WriteConcernError WriteConcernFailed", + "clientOptions": { + "retryWrites": true + }, + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "insert" + ], + "writeConcernError": { + "code": 64, + "codeName": "WriteConcernFailed", + "errmsg": "waiting for replication timed out", + "errInfo": { + "wtimeout": true + } + } + } + }, + "operation": { + "name": "insertOne", + "arguments": { + "document": { + "_id": 3, + "x": 33 + } + } + }, + "outcome": { + "error": true, + "collection": { + "data": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + } + } + ] +} diff --git a/t/data/retryable-writes/insertOne-serverErrors.yml b/t/data/retryable-writes/insertOne-serverErrors.yml new file mode 100644 index 00000000..69173b9c --- /dev/null +++ b/t/data/retryable-writes/insertOne-serverErrors.yml @@ -0,0 +1,471 @@ +data: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + +# TODO: this should change to 4.0 once 4.0.0 is released. +minServerVersion: '3.99' + +tests: + - + description: "InsertOne succeeds after connection failure" + clientOptions: + retryWrites: true + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["insert"] + closeConnection: true + operation: + name: "insertOne" + arguments: + document: { _id: 3, x: 33 } + outcome: + result: + insertedId: 3 + collection: + data: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } + - + description: "InsertOne succeeds after NotMaster" + clientOptions: + retryWrites: true + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["insert"] + errorCode: 10107 + closeConnection: false + operation: + name: "insertOne" + arguments: + document: { _id: 3, x: 33 } + outcome: + result: + insertedId: 3 + collection: + data: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } + - + description: "InsertOne succeeds after NotMasterOrSecondary" + clientOptions: + retryWrites: true + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["insert"] + errorCode: 13436 + closeConnection: false + operation: + name: "insertOne" + arguments: + document: { _id: 3, x: 33 } + outcome: + result: + insertedId: 3 + collection: + data: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } + - + description: "InsertOne succeeds after NotMasterNoSlaveOk" + clientOptions: + retryWrites: true + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["insert"] + errorCode: 13435 + closeConnection: false + operation: + name: "insertOne" + arguments: + document: { _id: 3, x: 33 } + outcome: + result: + insertedId: 3 + collection: + data: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } + - + description: "InsertOne succeeds after InterruptedDueToReplStateChange" + clientOptions: + retryWrites: true + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["insert"] + errorCode: 11602 + closeConnection: false + operation: + name: "insertOne" + arguments: + document: { _id: 3, x: 33 } + outcome: + result: + insertedId: 3 + collection: + data: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } + - + description: "InsertOne succeeds after InterruptedAtShutdown" + clientOptions: + retryWrites: true + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["insert"] + errorCode: 11600 + closeConnection: false + operation: + name: "insertOne" + arguments: + document: { _id: 3, x: 33 } + outcome: + result: + insertedId: 3 + collection: + data: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } + - + description: "InsertOne succeeds after PrimarySteppedDown" + clientOptions: + retryWrites: true + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["insert"] + errorCode: 189 + closeConnection: false + operation: + name: "insertOne" + arguments: + document: { _id: 3, x: 33 } + outcome: + result: + insertedId: 3 + collection: + data: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } + - + description: "InsertOne succeeds after ShutdownInProgress" + clientOptions: + retryWrites: true + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["insert"] + errorCode: 91 + closeConnection: false + operation: + name: "insertOne" + arguments: + document: { _id: 3, x: 33 } + outcome: + result: + insertedId: 3 + collection: + data: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } + - + description: "InsertOne succeeds after HostNotFound" + clientOptions: + retryWrites: true + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["insert"] + errorCode: 7 + closeConnection: false + operation: + name: "insertOne" + arguments: + document: { _id: 3, x: 33 } + outcome: + result: + insertedId: 3 + collection: + data: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } + - + description: "InsertOne succeeds after HostUnreachable" + clientOptions: + retryWrites: true + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["insert"] + errorCode: 6 + closeConnection: false + operation: + name: "insertOne" + arguments: + document: { _id: 3, x: 33 } + outcome: + result: + insertedId: 3 + collection: + data: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } + - + description: "InsertOne succeeds after SocketException" + clientOptions: + retryWrites: true + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["insert"] + errorCode: 9001 + closeConnection: false + operation: + name: "insertOne" + arguments: + document: { _id: 3, x: 33 } + outcome: + result: + insertedId: 3 + collection: + data: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } + - + description: "InsertOne succeeds after NetworkTimeout" + clientOptions: + retryWrites: true + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["insert"] + errorCode: 89 + closeConnection: false + operation: + name: "insertOne" + arguments: + document: { _id: 3, x: 33 } + outcome: + result: + insertedId: 3 + collection: + data: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } + - + description: "InsertOne fails after Interrupted" + clientOptions: + retryWrites: true + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["insert"] + errorCode: 11601 + closeConnection: false + operation: + name: "insertOne" + arguments: + document: { _id: 3, x: 33 } + outcome: + error: true + collection: + data: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - + description: "InsertOne succeeds after WriteConcernError InterruptedAtShutdown" + clientOptions: + retryWrites: true + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["insert"] + writeConcernError: + code: 11600 + errmsg: Replication is being shut down + operation: + name: "insertOne" + arguments: + document: { _id: 3, x: 33 } + outcome: + result: + insertedId: 3 + collection: + data: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } + - + description: "InsertOne succeeds after WriteConcernError InterruptedDueToReplStateChange" + clientOptions: + retryWrites: true + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["insert"] + writeConcernError: + code: 11602 + errmsg: Replication is being shut down + operation: + name: "insertOne" + arguments: + document: { _id: 3, x: 33 } + outcome: + result: + insertedId: 3 + collection: + data: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } + - + description: "InsertOne succeeds after WriteConcernError PrimarySteppedDown" + clientOptions: + retryWrites: true + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["insert"] + writeConcernError: + code: 189 + errmsg: Replication is being shut down + operation: + name: "insertOne" + arguments: + document: { _id: 3, x: 33 } + outcome: + result: + insertedId: 3 + collection: + data: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } + - + description: "InsertOne succeeds after WriteConcernError ShutdownInProgress" + clientOptions: + retryWrites: true + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["insert"] + writeConcernError: + code: 91 + errmsg: Replication is being shut down + operation: + name: "insertOne" + arguments: + document: { _id: 3, x: 33 } + outcome: + result: + insertedId: 3 + collection: + data: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } + - + description: "InsertOne fails after multiple retryable writeConcernErrors" + clientOptions: + retryWrites: true + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["insert"] + writeConcernError: + code: 91 + errmsg: Replication is being shut down + operation: + name: "insertOne" + arguments: + document: { _id: 3, x: 33 } + outcome: + error: true + collection: + data: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } # The write was still applied. + - + description: "InsertOne fails after WriteConcernError Interrupted" + clientOptions: + retryWrites: true + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["insert"] + writeConcernError: + code: 11601 + errmsg: operation was interrupted + operation: + name: "insertOne" + arguments: + document: { _id: 3, x: 33 } + outcome: + error: true + collection: + data: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } # The write was still applied. + - + description: "InsertOne fails after WriteConcernError WriteConcernFailed" + clientOptions: + retryWrites: true + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["insert"] + writeConcernError: + code: 64 + codeName: WriteConcernFailed + errmsg: waiting for replication timed out + errInfo: {wtimeout: True} + operation: + name: "insertOne" + arguments: + document: { _id: 3, x: 33 } + outcome: + error: true + collection: + data: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } # The write was still applied. diff --git a/t/data/retryable-writes/insertOne.json b/t/data/retryable-writes/insertOne.json index 58ad6cc6..123d51ed 100644 --- a/t/data/retryable-writes/insertOne.json +++ b/t/data/retryable-writes/insertOne.json @@ -13,7 +13,11 @@ "tests": [ { "description": "InsertOne is committed on first attempt", + "clientOptions": { + "retryWrites": true + }, "failPoint": { + "configureFailPoint": "onPrimaryTransactionalWrite", "mode": { "times": 1 } @@ -51,7 +55,11 @@ }, { "description": "InsertOne is not committed on first attempt", + "clientOptions": { + "retryWrites": true + }, "failPoint": { + "configureFailPoint": "onPrimaryTransactionalWrite", "mode": { "times": 1 }, @@ -92,7 +100,11 @@ }, { "description": "InsertOne is never committed", + "clientOptions": { + "retryWrites": true + }, "failPoint": { + "configureFailPoint": "onPrimaryTransactionalWrite", "mode": { "times": 2 }, diff --git a/t/data/retryable-writes/insertOne.yml b/t/data/retryable-writes/insertOne.yml index 5a303ddc..0e535ddb 100644 --- a/t/data/retryable-writes/insertOne.yml +++ b/t/data/retryable-writes/insertOne.yml @@ -7,7 +7,10 @@ minServerVersion: '3.6' tests: - description: "InsertOne is committed on first attempt" + clientOptions: + retryWrites: true failPoint: + configureFailPoint: onPrimaryTransactionalWrite mode: { times: 1 } operation: name: "insertOne" @@ -23,7 +26,10 @@ tests: - { _id: 3, x: 33 } - description: "InsertOne is not committed on first attempt" + clientOptions: + retryWrites: true failPoint: + configureFailPoint: onPrimaryTransactionalWrite mode: { times: 1 } data: { failBeforeCommitExceptionCode: 1 } operation: @@ -40,7 +46,10 @@ tests: - { _id: 3, x: 33 } - description: "InsertOne is never committed" + clientOptions: + retryWrites: true failPoint: + configureFailPoint: onPrimaryTransactionalWrite mode: { times: 2 } data: { failBeforeCommitExceptionCode: 1 } operation: diff --git a/t/data/retryable-writes/replaceOne-serverErrors.json b/t/data/retryable-writes/replaceOne-serverErrors.json new file mode 100644 index 00000000..b9c449a8 --- /dev/null +++ b/t/data/retryable-writes/replaceOne-serverErrors.json @@ -0,0 +1,116 @@ +{ + "data": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + } + ], + "minServerVersion": "3.99", + "tests": [ + { + "description": "ReplaceOne succeeds after PrimarySteppedDown", + "clientOptions": { + "retryWrites": true + }, + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "update" + ], + "errorCode": 189 + } + }, + "operation": { + "name": "replaceOne", + "arguments": { + "filter": { + "_id": 1 + }, + "replacement": { + "_id": 1, + "x": 111 + } + } + }, + "outcome": { + "result": { + "matchedCount": 1, + "modifiedCount": 1, + "upsertedCount": 0 + }, + "collection": { + "data": [ + { + "_id": 1, + "x": 111 + }, + { + "_id": 2, + "x": 22 + } + ] + } + } + }, + { + "description": "ReplaceOne succeeds after WriteConcernError ShutdownInProgress", + "clientOptions": { + "retryWrites": true + }, + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "update" + ], + "writeConcernError": { + "code": 91, + "errmsg": "Replication is being shut down" + } + } + }, + "operation": { + "name": "replaceOne", + "arguments": { + "filter": { + "_id": 1 + }, + "replacement": { + "_id": 1, + "x": 111 + } + } + }, + "outcome": { + "result": { + "matchedCount": 1, + "modifiedCount": 1, + "upsertedCount": 0 + }, + "collection": { + "data": [ + { + "_id": 1, + "x": 111 + }, + { + "_id": 2, + "x": 22 + } + ] + } + } + } + ] +} diff --git a/t/data/retryable-writes/replaceOne-serverErrors.yml b/t/data/retryable-writes/replaceOne-serverErrors.yml new file mode 100644 index 00000000..bd2bb0f3 --- /dev/null +++ b/t/data/retryable-writes/replaceOne-serverErrors.yml @@ -0,0 +1,58 @@ +data: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + +# TODO: this should change to 4.0 once 4.0.0 is released. +minServerVersion: '3.99' + +tests: + - + description: "ReplaceOne succeeds after PrimarySteppedDown" + clientOptions: + retryWrites: true + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["update"] + errorCode: 189 + operation: + name: "replaceOne" + arguments: + filter: { _id: 1 } + replacement: { _id: 1, x: 111 } + outcome: + result: + matchedCount: 1 + modifiedCount: 1 + upsertedCount: 0 + collection: + data: + - { _id: 1, x: 111 } + - { _id: 2, x: 22 } + - + description: "ReplaceOne succeeds after WriteConcernError ShutdownInProgress" + clientOptions: + retryWrites: true + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["update"] + writeConcernError: + code: 91 + errmsg: Replication is being shut down + operation: + name: "replaceOne" + arguments: + filter: { _id: 1 } + replacement: { _id: 1, x: 111 } + outcome: + result: + matchedCount: 1 + modifiedCount: 1 + upsertedCount: 0 + collection: + data: + - { _id: 1, x: 111 } + - { _id: 2, x: 22 } diff --git a/t/data/retryable-writes/replaceOne.json b/t/data/retryable-writes/replaceOne.json index f31966c3..6e9107b7 100644 --- a/t/data/retryable-writes/replaceOne.json +++ b/t/data/retryable-writes/replaceOne.json @@ -13,7 +13,11 @@ "tests": [ { "description": "ReplaceOne is committed on first attempt", + "clientOptions": { + "retryWrites": true + }, "failPoint": { + "configureFailPoint": "onPrimaryTransactionalWrite", "mode": { "times": 1 } @@ -52,7 +56,11 @@ }, { "description": "ReplaceOne is not committed on first attempt", + "clientOptions": { + "retryWrites": true + }, "failPoint": { + "configureFailPoint": "onPrimaryTransactionalWrite", "mode": { "times": 1 }, @@ -94,7 +102,11 @@ }, { "description": "ReplaceOne is never committed", + "clientOptions": { + "retryWrites": true + }, "failPoint": { + "configureFailPoint": "onPrimaryTransactionalWrite", "mode": { "times": 2 }, diff --git a/t/data/retryable-writes/replaceOne.yml b/t/data/retryable-writes/replaceOne.yml index 08fbda9a..1eed95ed 100644 --- a/t/data/retryable-writes/replaceOne.yml +++ b/t/data/retryable-writes/replaceOne.yml @@ -7,7 +7,10 @@ minServerVersion: '3.6' tests: - description: "ReplaceOne is committed on first attempt" + clientOptions: + retryWrites: true failPoint: + configureFailPoint: onPrimaryTransactionalWrite mode: { times: 1 } operation: name: "replaceOne" @@ -25,7 +28,10 @@ tests: - { _id: 2, x: 22 } - description: "ReplaceOne is not committed on first attempt" + clientOptions: + retryWrites: true failPoint: + configureFailPoint: onPrimaryTransactionalWrite mode: { times: 1 } data: { failBeforeCommitExceptionCode: 1 } operation: @@ -44,7 +50,10 @@ tests: - { _id: 2, x: 22 } - description: "ReplaceOne is never committed" + clientOptions: + retryWrites: true failPoint: + configureFailPoint: onPrimaryTransactionalWrite mode: { times: 2 } data: { failBeforeCommitExceptionCode: 1 } operation: diff --git a/t/data/retryable-writes/updateOne-serverErrors.json b/t/data/retryable-writes/updateOne-serverErrors.json new file mode 100644 index 00000000..84ebf877 --- /dev/null +++ b/t/data/retryable-writes/updateOne-serverErrors.json @@ -0,0 +1,118 @@ +{ + "data": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + } + ], + "minServerVersion": "3.99", + "tests": [ + { + "description": "UpdateOne succeeds after PrimarySteppedDown", + "clientOptions": { + "retryWrites": true + }, + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "update" + ], + "errorCode": 189 + } + }, + "operation": { + "name": "updateOne", + "arguments": { + "filter": { + "_id": 1 + }, + "update": { + "$inc": { + "x": 1 + } + } + } + }, + "outcome": { + "result": { + "matchedCount": 1, + "modifiedCount": 1, + "upsertedCount": 0 + }, + "collection": { + "data": [ + { + "_id": 1, + "x": 12 + }, + { + "_id": 2, + "x": 22 + } + ] + } + } + }, + { + "description": "UpdateOne succeeds after WriteConcernError ShutdownInProgress", + "clientOptions": { + "retryWrites": true + }, + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "update" + ], + "writeConcernError": { + "code": 91, + "errmsg": "Replication is being shut down" + } + } + }, + "operation": { + "name": "updateOne", + "arguments": { + "filter": { + "_id": 1 + }, + "update": { + "$inc": { + "x": 1 + } + } + } + }, + "outcome": { + "result": { + "matchedCount": 1, + "modifiedCount": 1, + "upsertedCount": 0 + }, + "collection": { + "data": [ + { + "_id": 1, + "x": 12 + }, + { + "_id": 2, + "x": 22 + } + ] + } + } + } + ] +} diff --git a/t/data/retryable-writes/updateOne-serverErrors.yml b/t/data/retryable-writes/updateOne-serverErrors.yml new file mode 100644 index 00000000..1d468861 --- /dev/null +++ b/t/data/retryable-writes/updateOne-serverErrors.yml @@ -0,0 +1,58 @@ +data: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + +# TODO: this should change to 4.0 once 4.0.0 is released. +minServerVersion: '3.99' + +tests: + - + description: "UpdateOne succeeds after PrimarySteppedDown" + clientOptions: + retryWrites: true + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["update"] + errorCode: 189 + operation: + name: "updateOne" + arguments: + filter: { _id: 1 } + update: { $inc: { x : 1 }} + outcome: + result: + matchedCount: 1 + modifiedCount: 1 + upsertedCount: 0 + collection: + data: + - { _id: 1, x: 12 } + - { _id: 2, x: 22 } + - + description: "UpdateOne succeeds after WriteConcernError ShutdownInProgress" + clientOptions: + retryWrites: true + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["update"] + writeConcernError: + code: 91 + errmsg: Replication is being shut down + operation: + name: "updateOne" + arguments: + filter: { _id: 1 } + update: { $inc: { x : 1 }} + outcome: + result: + matchedCount: 1 + modifiedCount: 1 + upsertedCount: 0 + collection: + data: + - { _id: 1, x: 12 } + - { _id: 2, x: 22 } diff --git a/t/data/retryable-writes/updateOne.json b/t/data/retryable-writes/updateOne.json index 96aa2bde..c342c562 100644 --- a/t/data/retryable-writes/updateOne.json +++ b/t/data/retryable-writes/updateOne.json @@ -13,7 +13,11 @@ "tests": [ { "description": "UpdateOne is committed on first attempt", + "clientOptions": { + "retryWrites": true + }, "failPoint": { + "configureFailPoint": "onPrimaryTransactionalWrite", "mode": { "times": 1 } @@ -53,7 +57,11 @@ }, { "description": "UpdateOne is not committed on first attempt", + "clientOptions": { + "retryWrites": true + }, "failPoint": { + "configureFailPoint": "onPrimaryTransactionalWrite", "mode": { "times": 1 }, @@ -96,7 +104,11 @@ }, { "description": "UpdateOne is never committed", + "clientOptions": { + "retryWrites": true + }, "failPoint": { + "configureFailPoint": "onPrimaryTransactionalWrite", "mode": { "times": 2 }, @@ -135,7 +147,11 @@ }, { "description": "UpdateOne with upsert is committed on first attempt", + "clientOptions": { + "retryWrites": true + }, "failPoint": { + "configureFailPoint": "onPrimaryTransactionalWrite", "mode": { "times": 1 } @@ -182,7 +198,11 @@ }, { "description": "UpdateOne with upsert is not committed on first attempt", + "clientOptions": { + "retryWrites": true + }, "failPoint": { + "configureFailPoint": "onPrimaryTransactionalWrite", "mode": { "times": 1 }, @@ -232,7 +252,11 @@ }, { "description": "UpdateOne with upsert is never committed", + "clientOptions": { + "retryWrites": true + }, "failPoint": { + "configureFailPoint": "onPrimaryTransactionalWrite", "mode": { "times": 2 }, diff --git a/t/data/retryable-writes/updateOne.yml b/t/data/retryable-writes/updateOne.yml index 810e49f2..900d4e7f 100644 --- a/t/data/retryable-writes/updateOne.yml +++ b/t/data/retryable-writes/updateOne.yml @@ -7,7 +7,10 @@ minServerVersion: '3.6' tests: - description: "UpdateOne is committed on first attempt" + clientOptions: + retryWrites: true failPoint: + configureFailPoint: onPrimaryTransactionalWrite mode: { times: 1 } operation: name: "updateOne" @@ -25,7 +28,10 @@ tests: - { _id: 2, x: 22 } - description: "UpdateOne is not committed on first attempt" + clientOptions: + retryWrites: true failPoint: + configureFailPoint: onPrimaryTransactionalWrite mode: { times: 1 } data: { failBeforeCommitExceptionCode: 1 } operation: @@ -44,7 +50,10 @@ tests: - { _id: 2, x: 22 } - description: "UpdateOne is never committed" + clientOptions: + retryWrites: true failPoint: + configureFailPoint: onPrimaryTransactionalWrite mode: { times: 2 } data: { failBeforeCommitExceptionCode: 1 } operation: @@ -60,7 +69,10 @@ tests: - { _id: 2, x: 22 } - description: "UpdateOne with upsert is committed on first attempt" + clientOptions: + retryWrites: true failPoint: + configureFailPoint: onPrimaryTransactionalWrite mode: { times: 1 } operation: name: "updateOne" @@ -81,7 +93,10 @@ tests: - { _id: 3, x: 34 } - description: "UpdateOne with upsert is not committed on first attempt" + clientOptions: + retryWrites: true failPoint: + configureFailPoint: onPrimaryTransactionalWrite mode: { times: 1 } data: { failBeforeCommitExceptionCode: 1 } operation: @@ -103,7 +118,10 @@ tests: - { _id: 3, x: 34 } - description: "UpdateOne with upsert is never committed" + clientOptions: + retryWrites: true failPoint: + configureFailPoint: onPrimaryTransactionalWrite mode: { times: 2 } data: { failBeforeCommitExceptionCode: 1 } operation: From da823c716abd9c6e804c22ff7cd6b83f0184e1a6 Mon Sep 17 00:00:00 2001 From: Thomas Bloor Date: Mon, 18 Jun 2018 17:43:31 +0100 Subject: [PATCH 2/6] minor: Added Callback class to factor out basic monitoring callback work --- t/lib/MongoDBTest/Callback.pm | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 t/lib/MongoDBTest/Callback.pm diff --git a/t/lib/MongoDBTest/Callback.pm b/t/lib/MongoDBTest/Callback.pm new file mode 100644 index 00000000..e87980d0 --- /dev/null +++ b/t/lib/MongoDBTest/Callback.pm @@ -0,0 +1,22 @@ +package MongoDBTest::Callback; + +use Moo; +use Storable qw/ dclone /; + +has events => ( + is => 'lazy', + default => sub { [] }, + clearer => 1, +); + +sub callback { + my $self = shift; + return sub { push @{ $self->events }, dclone $_[0] }; +} + +sub count { + my $self = shift; + return scalar( @{ $self->events } ); +} + +1; From a92e5c0bb6017f2e3a84e3103b4ba0fd7d48463b Mon Sep 17 00:00:00 2001 From: Thomas Bloor Date: Mon, 18 Jun 2018 17:44:31 +0100 Subject: [PATCH 3/6] PERL-918 - refactored spec tests for retryable writes --- t/lib/MongoDBTest.pm | 18 ++++++++ t/retryable-writes-spec.t | 91 +++++++++++++++++---------------------- 2 files changed, 57 insertions(+), 52 deletions(-) diff --git a/t/lib/MongoDBTest.pm b/t/lib/MongoDBTest.pm index f9cd0871..83ca55a4 100644 --- a/t/lib/MongoDBTest.pm +++ b/t/lib/MongoDBTest.pm @@ -36,6 +36,8 @@ our @EXPORT_OK = qw( skip_unless_mongod skip_unless_failpoints_available skip_unless_sessions + to_snake_case + remap_hashref_to_snake_case uri_escape get_unique_collection get_features @@ -271,6 +273,22 @@ sub uri_escape { return $str; } +sub to_snake_case { + my $t = shift; + $t =~ s{([A-Z])}{_\L$1}g; + return $t; +} + +sub remap_hashref_to_snake_case { + my $hash = shift; + return { + map { + my $k = to_snake_case( $_ ); + $k => $hash->{ $_ } + } keys %$hash + } +} + sub uuid_to_string { my $uuid = shift; return join "-", unpack( "H8H4H4H4H12", $uuid ); diff --git a/t/retryable-writes-spec.t b/t/retryable-writes-spec.t index 44d87c62..549c0e32 100644 --- a/t/retryable-writes-spec.t +++ b/t/retryable-writes-spec.t @@ -17,7 +17,8 @@ use warnings; use JSON::MaybeXS; use Path::Tiny 0.054; # basename with suffix use Test::More 0.88; -use Test::Fatal; +use Test::Deep ':v1'; +use Safe::Isa; use lib "t/lib"; @@ -32,6 +33,8 @@ use MongoDBTest qw/ skip_unless_mongod skip_unless_sessions skip_unless_failpoints_available + to_snake_case + remap_hashref_to_snake_case get_features /; @@ -46,7 +49,7 @@ my $server_type = server_type($conn); sub run_test { my ( $coll, $test ) = @_; - enable_failpoint( $test->{failPoint} ) if exists $test->{failPoint}; + enable_failpoint( $test->{failPoint} ); my $op = $test->{operation}; my $method = $op->{name}; @@ -63,33 +66,13 @@ sub run_test { if ( !exists $test->{outcome}{error} && exists $test->{outcome}->{result} ) { - #Dwarn $ret; - #Dwarn $test->{outcome}; - for my $res_key ( keys %{ $test->{outcome}->{result} } ) { - next if $res_key eq 'upsertedCount' && ! $ret->can('upserted_count'); # Driver does not parse this value on all things? - # next if $res_key eq 'upsertedId' && ! defined $ret->upserted_id; # upserted id is always present - my $res = $test->{outcome}->{result}->{$res_key}; - - if ( $res_key eq 'insertedIds' ) { - my $ret_parsed = {}; - for my $item ( @{ $ret->inserted } ) { - $ret_parsed->{$item->{index}} = $item->{_id}; - } - is_deeply $ret_parsed, $test->{outcome}->{result}->{insertedIds}, 'insertedIds correct in result'; - next; - } - if ( $res_key eq 'upsertedIds' ) { - my $ret_parsed = {}; - for my $item ( @{ $ret->upserted } ) { - $ret_parsed->{$item->{index}} = $item->{_id}; - } - is_deeply $ret_parsed, $test->{outcome}->{result}->{upsertedIds}, 'upsertedIds correct in result'; - next; - } - my $ret_key = $res_key; - $ret_key =~ s{([A-Z])}{_\L$1}g; + my $expected = remap_hashref_to_snake_case( $test->{outcome}->{result} ); + # not all commands return an upserted count + delete $expected->{upserted_count} unless $ret->$_can('upserted_count'); - is $ret->{$ret_key}, $res, "$res_key correct in result"; + for my $key ( keys %$expected ) { + my $got = ref $ret eq 'HASH' ? $ret->{$key} : $ret->$key; + cmp_deeply $got, $expected->{$key}, "$key result as expected"; } } @@ -97,7 +80,7 @@ sub run_test { my $coll_expected = $test->{outcome}->{collection}->{data}; is_deeply \@coll_outcome, $coll_expected, 'Collection has correct outcome'; - disable_failpoint() if exists $test->{failPoint}; + disable_failpoint( $test->{failPoint} ); } sub do_delete_one { @@ -152,6 +135,15 @@ sub do_find_one_and_delete { return $coll->find_one_and_delete( $filter, $options ); } +my %bulk_remap = ( + insert_one => [qw( document )], + update_one => [qw( filter update )], + update_many => [qw( filter update )], + replace_one => [qw( filter replacement )], + delete_one => [qw( filter )], + delete_many => [qw( filter )], +); + sub do_bulk_write { my ( $self, $coll, $args ) = @_; my $options = { @@ -164,24 +156,13 @@ sub do_bulk_write { my @arguments; for my $request ( @{ $args->{requests} } ) { - if ( $request->{name} eq 'insertOne' ) { - push @arguments, { insert_one => [ $request->{arguments}->{document} ] }; - } elsif ( $request->{name} eq 'updateOne' ) { - push @arguments, { update_one => [ - $request->{arguments}->{filter}, - $request->{arguments}->{update}, - ( defined $request->{arguments}->{upsert} - ? ( { upsert => $request->{arguments}->{upsert} ? 1 : 0 } ) - : () ) - ] }; - } elsif ( $request->{name} eq 'deleteOne' ) { - push @arguments, { delete_one => [ $request->{arguments}->{filter} ] }; - } elsif ( $request->{name} eq 'replaceOne' ) { - push @arguments, { replace_one => [ - $request->{arguments}->{filter}, - $request->{arguments}->{replacement} - ] }; - } + my $req_name = to_snake_case( $request->{name} ); + my @req_fields = @{ $bulk_remap{ $req_name } }; + my @arg = map { + delete $request->{arguments}->{ $_ } + } @req_fields; + push @arg, $request->{arguments} if keys %{ $request->{arguments} }; + push @arguments, { $req_name => \@arg }; } return $coll->bulk_write( \@arguments, $options ); } @@ -230,6 +211,7 @@ while ( my $path = $iterator->() ) { my $coll = get_unique_collection( $testdb, 'retry_write' ); my $ret = $coll->insert_many( $plan->{data} ); my $description = $test->{description}; + subtest $description => sub { run_test( $coll, $test ); } @@ -238,17 +220,22 @@ while ( my $path = $iterator->() ) { } sub enable_failpoint { - my $doc = shift; + my $failpoint = shift; + return unless defined $failpoint; $conn->send_admin_command([ - configureFailPoint => 'onPrimaryTransactionalWrite', - %$doc, + configureFailPoint => $failpoint->{configureFailPoint}, + mode => $failpoint->{mode}, + defined $failpoint->{data} + ? ( data => $failpoint->{data} ) + : (), ]); } sub disable_failpoint { - my $doc = shift; + my $failpoint = shift; + return unless defined $failpoint; $conn->send_admin_command([ - configureFailPoint => 'onPrimaryTransactionalWrite', + configureFailPoint => $failpoint->{configureFailPoint}, mode => 'off', ]); } From b8a0dadfb258676cbfe09243b5d6ecbbd256248f Mon Sep 17 00:00:00 2001 From: Thomas Bloor Date: Mon, 18 Jun 2018 18:01:25 +0100 Subject: [PATCH 4/6] PERL-918 Add new error names and codes --- lib/MongoDB/Error.pm | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/MongoDB/Error.pm b/lib/MongoDB/Error.pm index 4e76cd8d..c9399f76 100644 --- a/lib/MongoDB/Error.pm +++ b/lib/MongoDB/Error.pm @@ -40,6 +40,8 @@ my $ERROR_CODES; BEGIN { $ERROR_CODES = { BAD_VALUE => 2, + HOST_UNREACHABLE => 6, + HOST_NOT_FOUND => 7, UNKNOWN_ERROR => 8, USER_NOT_FOUND => 11, NAMESPACE_NOT_FOUND => 26, @@ -48,9 +50,15 @@ BEGIN { EXCEEDED_TIME_LIMIT => 50, COMMAND_NOT_FOUND => 59, WRITE_CONCERN_ERROR => 64, + NETWORK_TIMEOUT => 89, + SHUTDOWN_IN_PROGRESS => 91, + PRIMARY_STEPPED_DOWN => 189, + SOCKET_EXCEPTION => 9001, NOT_MASTER => 10107, DUPLICATE_KEY => 11000, DUPLICATE_KEY_UPDATE => 11001, # legacy before 2.6 + INTERRUPTED_AT_SHUTDOWN => 11600, + INTERRUPTED_DUE_TO_REPL_STATE_CHANGE => 11602, DUPLICATE_KEY_CAPPED => 12582, # legacy before 2.6 UNRECOGNIZED_COMMAND => 13390, # mongos error before 2.4 NOT_MASTER_NO_SLAVE_OK => 13435, From 94b8a0240bb4a9e40ecba5b6d34fb9fb26afdc4a Mon Sep 17 00:00:00 2001 From: Thomas Bloor Date: Tue, 19 Jun 2018 18:28:09 +0100 Subject: [PATCH 5/6] PERL-918 Finish updating tests for retryable-writes --- lib/MongoDB/Error.pm | 60 ++++++++++++++++++++++++ lib/MongoDB/MongoClient.pm | 3 +- lib/MongoDB/Op/_Command.pm | 2 + lib/MongoDB/Role/_SessionSupport.pm | 14 ++++++ lib/MongoDB/Role/_SingleBatchDocWrite.pm | 4 +- 5 files changed, 81 insertions(+), 2 deletions(-) diff --git a/lib/MongoDB/Error.pm b/lib/MongoDB/Error.pm index c9399f76..e6c77409 100644 --- a/lib/MongoDB/Error.pm +++ b/lib/MongoDB/Error.pm @@ -32,6 +32,7 @@ use MongoDB::_Types qw( ); use Scalar::Util (); use Sub::Quote (); +use Safe::Isa; use Exporter 5.57 qw/import/; use namespace::clean -except => ['import']; @@ -121,6 +122,62 @@ sub throw { # an error occurs. sub _is_resumable { 1 } +# internal flag for if this error type specifically can be retried regardless +# of other state. See _is_retryable which contains the full retryable error +# logic. +sub __is_retryable_error { 0 } + +sub _check_is_retryable_code { + my $code = $_[-1]; + + my @retryable_codes = ( + MongoDB::Error::HOST_NOT_FOUND(), + MongoDB::Error::HOST_UNREACHABLE(), + MongoDB::Error::NETWORK_TIMEOUT(), + MongoDB::Error::SHUTDOWN_IN_PROGRESS(), + MongoDB::Error::PRIMARY_STEPPED_DOWN(), + MongoDB::Error::SOCKET_EXCEPTION(), + MongoDB::Error::NOT_MASTER(), + MongoDB::Error::INTERRUPTED_AT_SHUTDOWN(), + MongoDB::Error::INTERRUPTED_DUE_TO_REPL_STATE_CHANGE(), + MongoDB::Error::NOT_MASTER_NO_SLAVE_OK(), + MongoDB::Error::NOT_MASTER_OR_SECONDARY(), + ); + + return 1 if grep { $code == $_ } @retryable_codes; + return 0; +} + +sub _check_is_retryable_message { + my $message = $_[-1]; + + return 1 if $message =~ /(not master|node is recovering)/i; + return 0; +} + +# indicates if this error can be retried under retryable writes +sub _is_retryable { + my $self = shift; + + if ( $self->$_can( 'result' ) ) { + return 1 if _check_is_retryable_code( $self->result->last_code ); + } + + if ( $self->$_can( 'code' ) ) { + return 1 if _check_is_retryable_code( $self->code ); + } + + return 1 if _check_is_retryable_message( $self->message ); + + if ( $self->$_isa( 'MongoDB::WriteConcernError' ) ) { + return 1 if _check_is_retryable_code( $self->result->output->{writeConcernError}{code} ); + return 1 if _check_is_retryable_message( $self->result->output->{writeConcernError}{message} ); + } + + # Defaults to 0 unless its a network exception + return $self->__is_retryable_error; +} + #--------------------------------------------------------------------------# # Subclasses with attributes included inline below #--------------------------------------------------------------------------# @@ -208,6 +265,7 @@ use namespace::clean; extends 'MongoDB::Error'; sub _is_resumable { 1 } +sub __is_retryable_error { 1 } package MongoDB::HandshakeError; use Moo; @@ -225,6 +283,8 @@ use Moo; use namespace::clean; extends 'MongoDB::Error'; +sub __is_retryable_error { 1 } + package MongoDB::ExecutionTimeout; use Moo; use namespace::clean; diff --git a/lib/MongoDB/MongoClient.pm b/lib/MongoDB/MongoClient.pm index 0919f6f2..ebccffd2 100644 --- a/lib/MongoDB/MongoClient.pm +++ b/lib/MongoDB/MongoClient.pm @@ -1624,7 +1624,8 @@ sub send_retryable_write_op { my $retry_link = $self->{_topology}->get_writable_link; # Rare chance that the new link is not retryable - unless ( $retry_link->supports_retryWrites ) { + unless ( $retry_link->supports_retryWrites + && $err->$_call_if_can('_is_retryable') ) { WITH_ASSERTS ? ( confess $err ) : ( die $err ); } diff --git a/lib/MongoDB/Op/_Command.pm b/lib/MongoDB/Op/_Command.pm index eed6e74b..fd3d9955 100644 --- a/lib/MongoDB/Op/_Command.pm +++ b/lib/MongoDB/Op/_Command.pm @@ -130,6 +130,8 @@ sub execute { $self->_update_session_and_cluster_time($res); + $self->_assert_session_errors($res); + return $res; } diff --git a/lib/MongoDB/Role/_SessionSupport.pm b/lib/MongoDB/Role/_SessionSupport.pm index 8e76a2cb..72f33c85 100644 --- a/lib/MongoDB/Role/_SessionSupport.pm +++ b/lib/MongoDB/Role/_SessionSupport.pm @@ -92,6 +92,20 @@ sub _update_operation_time { return; } +# Certain errors have to happen as soon as possible, such as write concern +# errors in a retryable write. This has to be seperate to the other functions +# due to not all result objects having the base response inside, so cannot be +# used to parse operationTime or $clusterTime +sub _assert_session_errors { + my ( $self, $response ) = @_; + + if ( $self->retryable_write ) { + $response->assert_no_write_concern_error; + } + + return; +} + sub __extract_from { my ( $self, $response, $key ) = @_; diff --git a/lib/MongoDB/Role/_SingleBatchDocWrite.pm b/lib/MongoDB/Role/_SingleBatchDocWrite.pm index 70430f22..abbc470a 100644 --- a/lib/MongoDB/Role/_SingleBatchDocWrite.pm +++ b/lib/MongoDB/Role/_SingleBatchDocWrite.pm @@ -222,12 +222,14 @@ sub _send_write_command { # otherwise, construct the desired result object, calling back # on class-specific parser to generate additional attributes - return $result_class->_new( + my $built_result = $result_class->_new( write_errors => ( $res->{writeErrors} ? $res->{writeErrors} : [] ), write_concern_errors => ( $res->{writeConcernError} ? [ $res->{writeConcernError} ] : [] ), $self->_parse_cmd($res), ); + $self->_assert_session_errors( $built_result ); + return $built_result; } else { return MongoDB::UnacknowledgedResult->_new( From 9ba406d402b6d1236daf7c14de7c54d3b0ca7472 Mon Sep 17 00:00:00 2001 From: Thomas Bloor Date: Thu, 21 Jun 2018 16:30:31 +0100 Subject: [PATCH 6/6] PERL-918 re-arrange retry error logic and add clearer comments --- lib/MongoDB/MongoClient.pm | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/MongoDB/MongoClient.pm b/lib/MongoDB/MongoClient.pm index ebccffd2..53188264 100644 --- a/lib/MongoDB/MongoClient.pm +++ b/lib/MongoDB/MongoClient.pm @@ -1621,11 +1621,18 @@ sub send_retryable_write_op { # attempt the op the first time eval { ($result) = $self->_try_write_op_for_link( $link, $op ); 1 } or do { my $err = length($@) ? $@ : "caught error, but it was lost in eval unwind"; + + # If the error is not retryable, then drop out + unless ( $err->$_call_if_can('_is_retryable') ) { + WITH_ASSERTS ? ( confess $err ) : ( die $err ); + } + + # Must check if error is retryable before getting the link, in case we + # get a 'no writable servers' error my $retry_link = $self->{_topology}->get_writable_link; # Rare chance that the new link is not retryable - unless ( $retry_link->supports_retryWrites - && $err->$_call_if_can('_is_retryable') ) { + unless ( $retry_link->supports_retryWrites ) { WITH_ASSERTS ? ( confess $err ) : ( die $err ); }