PERL-918 Update retryable writes spec

lib/MongoDB/Error.pm

@@ -32,6 +32,7 @@ use MongoDB::_Types qw(
 );
 use Scalar::Util ();
 use Sub::Quote ();
+use Safe::Isa;
 use Exporter 5.57 qw/import/;
 use namespace::clean -except => ['import'];
 
@@ -40,6 +41,8 @@ my $ERROR_CODES;
 BEGIN {
     $ERROR_CODES = {
         BAD_VALUE                 => 2,
+        HOST_UNREACHABLE          => 6,
+        HOST_NOT_FOUND            => 7,
         UNKNOWN_ERROR             => 8,
         USER_NOT_FOUND            => 11,
         NAMESPACE_NOT_FOUND       => 26,
@@ -48,9 +51,15 @@ BEGIN {
         EXCEEDED_TIME_LIMIT       => 50,
         COMMAND_NOT_FOUND         => 59,
         WRITE_CONCERN_ERROR       => 64,
+        NETWORK_TIMEOUT           => 89,
+        SHUTDOWN_IN_PROGRESS      => 91,
+        PRIMARY_STEPPED_DOWN      => 189,
+        SOCKET_EXCEPTION          => 9001,
         NOT_MASTER                => 10107,
         DUPLICATE_KEY             => 11000,
         DUPLICATE_KEY_UPDATE      => 11001, # legacy before 2.6
+        INTERRUPTED_AT_SHUTDOWN   => 11600,
+        INTERRUPTED_DUE_TO_REPL_STATE_CHANGE => 11602,
         DUPLICATE_KEY_CAPPED      => 12582, # legacy before 2.6
         UNRECOGNIZED_COMMAND      => 13390, # mongos error before 2.4
         NOT_MASTER_NO_SLAVE_OK    => 13435,
@@ -113,6 +122,62 @@ sub throw {
 # an error occurs.
 sub _is_resumable { 1 }
 
+# internal flag for if this error type specifically can be retried regardless
+# of other state. See _is_retryable which contains the full retryable error
+# logic.
+sub __is_retryable_error { 0 }
+
+sub _check_is_retryable_code {
+    my $code = $_[-1];
+
+    my @retryable_codes = (
+        MongoDB::Error::HOST_NOT_FOUND(),
+        MongoDB::Error::HOST_UNREACHABLE(),
+        MongoDB::Error::NETWORK_TIMEOUT(),
+        MongoDB::Error::SHUTDOWN_IN_PROGRESS(),
+        MongoDB::Error::PRIMARY_STEPPED_DOWN(),
+        MongoDB::Error::SOCKET_EXCEPTION(),
+        MongoDB::Error::NOT_MASTER(),
+        MongoDB::Error::INTERRUPTED_AT_SHUTDOWN(),
+        MongoDB::Error::INTERRUPTED_DUE_TO_REPL_STATE_CHANGE(),
+        MongoDB::Error::NOT_MASTER_NO_SLAVE_OK(),
+        MongoDB::Error::NOT_MASTER_OR_SECONDARY(),
+    );
+
+    return 1 if grep { $code == $_ } @retryable_codes;
+    return 0;
+}
+
+sub _check_is_retryable_message {
+  my $message = $_[-1];
+
+  return 1 if $message =~ /(not master|node is recovering)/i;
+  return 0;
+}
+
+# indicates if this error can be retried under retryable writes
+sub _is_retryable {
+    my $self = shift;
+
+    if ( $self->$_can( 'result' ) ) {
+        return 1 if _check_is_retryable_code( $self->result->last_code );
+    }
+
+    if ( $self->$_can( 'code' ) ) {
+        return 1 if _check_is_retryable_code( $self->code );
+    }
+
+    return 1 if _check_is_retryable_message( $self->message );
+
+    if ( $self->$_isa( 'MongoDB::WriteConcernError' ) ) {
+      return 1 if _check_is_retryable_code( $self->result->output->{writeConcernError}{code} );
+      return 1 if _check_is_retryable_message( $self->result->output->{writeConcernError}{message} );
+    }
+
+    # Defaults to 0 unless its a network exception
+    return $self->__is_retryable_error;
+}
+
 #--------------------------------------------------------------------------#
 # Subclasses with attributes included inline below
 #--------------------------------------------------------------------------#
@@ -200,6 +265,7 @@ use namespace::clean;
 extends 'MongoDB::Error';
 
 sub _is_resumable { 1 }
+sub __is_retryable_error { 1 }
 
 package MongoDB::HandshakeError;
 use Moo;
@@ -217,6 +283,8 @@ use Moo;
 use namespace::clean;
 extends 'MongoDB::Error';
 
+sub __is_retryable_error { 1 }
+
 package MongoDB::ExecutionTimeout;
 use Moo;
 use namespace::clean;

lib/MongoDB/MongoClient.pm

@@ -1621,6 +1621,14 @@ sub send_retryable_write_op {
     # attempt the op the first time
     eval { ($result) = $self->_try_write_op_for_link( $link, $op ); 1 } or do {
         my $err = length($@) ? $@ : "caught error, but it was lost in eval unwind";
+
+        # If the error is not retryable, then drop out
+        unless ( $err->$_call_if_can('_is_retryable') ) {
+            WITH_ASSERTS ? ( confess $err ) : ( die $err );
+        }
+
+        # Must check if error is retryable before getting the link, in case we
+        # get a 'no writable servers' error
         my $retry_link = $self->{_topology}->get_writable_link;
 
         # Rare chance that the new link is not retryable

lib/MongoDB/Op/_Command.pm

@@ -130,6 +130,8 @@ sub execute {
 
     $self->_update_session_and_cluster_time($res);
 
+    $self->_assert_session_errors($res);
+
     return $res;
 }
 

lib/MongoDB/Role/_SessionSupport.pm

@@ -92,6 +92,20 @@ sub _update_operation_time {
     return;
 }
 
+# Certain errors have to happen as soon as possible, such as write concern
+# errors in a retryable write. This has to be seperate to the other functions
+# due to not all result objects having the base response inside, so cannot be
+# used to parse operationTime or $clusterTime
+sub _assert_session_errors {
+    my ( $self, $response ) = @_;
+
+    if ( $self->retryable_write ) {
+        $response->assert_no_write_concern_error;
+    }
+
+    return;
+}
+
 sub __extract_from {
     my ( $self, $response, $key ) = @_;
 

lib/MongoDB/Role/_SingleBatchDocWrite.pm

@@ -222,12 +222,14 @@ sub _send_write_command {
 
         # otherwise, construct the desired result object, calling back
         # on class-specific parser to generate additional attributes
-        return $result_class->_new(
+        my $built_result = $result_class->_new(
             write_errors => ( $res->{writeErrors} ? $res->{writeErrors} : [] ),
             write_concern_errors =>
               ( $res->{writeConcernError} ? [ $res->{writeConcernError} ] : [] ),
             $self->_parse_cmd($res),
         );
+        $self->_assert_session_errors( $built_result );
+        return $built_result;
     }
     else {
         return MongoDB::UnacknowledgedResult->_new(

t/data/retryable-writes/README.rst

@@ -15,10 +15,11 @@ that drivers can use to prove their conformance to the Retryable Writes spec.
 Several prose tests, which are not easily expressed in YAML, are also presented
 in this file. Those tests will need to be manually implemented by each driver.
 
-Tests will require a MongoClient with ``retryWrites`` enabled. Integration tests
-will require a running MongoDB cluster with server versions 3.6.0 or later. The
-``{setFeatureCompatibilityVersion: 3.6}`` admin command will also need to have
-been executed to enable support for retryable writes on the cluster.
+Tests will require a MongoClient created with options defined in the tests.
+Integration tests will require a running MongoDB cluster with server versions
+3.6.0 or later. The ``{setFeatureCompatibilityVersion: 3.6}`` admin command
+will also need to have been executed to enable support for retryable writes on
+the cluster.
 
 Server Fail Point
 =================
@@ -137,15 +138,16 @@ Each YAML file has the following keys:
 
   - ``description``: The name of the test.
 
-  - ``failPoint``: Document describing options for configuring the
-    ``onPrimaryTransactionalWrite`` fail point on the primary server. This
-    document should be merged with the
-    ``{ configureFailPoint: "onPrimaryTransactionalWrite" }`` command document.
+  - ``clientOptions``: Parameters to pass to MongoClient().
+
+  - ``failPoint``: The ``configureFailPoint`` command document to run to
+    configure a fail point on the primary server. Drivers must ensure that
+    ``configureFailPoint`` is the first field in the command.
 
   - ``operation``: Document describing the operation to be executed. The
     operation should be executed through a collection object derived from a
-    client that has been created with the ``retryWrites=true`` option.
-    This will have some or all of the following fields:
+    client that has been created with ``clientOptions``. The operation will have
+    some or all of the following fields:
 
     - ``name``: The name of the operation as defined in the CRUD specification.
 

t/data/retryable-writes/bulkWrite-serverErrors.json

@@ -0,0 +1,182 @@
+{
+  "data": [
+    {
+      "_id": 1,
+      "x": 11
+    },
+    {
+      "_id": 2,
+      "x": 22
+    }
+  ],
+  "minServerVersion": "3.99",
+  "tests": [
+    {
+      "description": "BulkWrite succeeds after PrimarySteppedDown",
+      "clientOptions": {
+        "retryWrites": true
+      },
+      "failPoint": {
+        "configureFailPoint": "failCommand",
+        "mode": {
+          "times": 1
+        },
+        "data": {
+          "failCommands": [
+            "update"
+          ],
+          "errorCode": 189
+        }
+      },
+      "operation": {
+        "name": "bulkWrite",
+        "arguments": {
+          "requests": [
+            {
+              "name": "deleteOne",
+              "arguments": {
+                "filter": {
+                  "_id": 1
+                }
+              }
+            },
+            {
+              "name": "insertOne",
+              "arguments": {
+                "document": {
+                  "_id": 3,
+                  "x": 33
+                }
+              }
+            },
+            {
+              "name": "updateOne",
+              "arguments": {
+                "filter": {
+                  "_id": 2
+                },
+                "update": {
+                  "$inc": {
+                    "x": 1
+                  }
+                }
+              }
+            }
+          ],
+          "options": {
+            "ordered": true
+          }
+        }
+      },
+      "outcome": {
+        "result": {
+          "deletedCount": 1,
+          "insertedIds": {
+            "1": 3
+          },
+          "matchedCount": 1,
+          "modifiedCount": 1,
+          "upsertedCount": 0,
+          "upsertedIds": {}
+        },
+        "collection": {
+          "data": [
+            {
+              "_id": 2,
+              "x": 23
+            },
+            {
+              "_id": 3,
+              "x": 33
+            }
+          ]
+        }
+      }
+    },
+    {
+      "description": "BulkWrite succeeds after WriteConcernError ShutdownInProgress",
+      "clientOptions": {
+        "retryWrites": true
+      },
+      "failPoint": {
+        "configureFailPoint": "failCommand",
+        "mode": {
+          "times": 1
+        },
+        "data": {
+          "failCommands": [
+            "insert"
+          ],
+          "writeConcernError": {
+            "code": 91,
+            "errmsg": "Replication is being shut down"
+          }
+        }
+      },
+      "operation": {
+        "name": "bulkWrite",
+        "arguments": {
+          "requests": [
+            {
+              "name": "deleteOne",
+              "arguments": {
+                "filter": {
+                  "_id": 1
+                }
+              }
+            },
+            {
+              "name": "insertOne",
+              "arguments": {
+                "document": {
+                  "_id": 3,
+                  "x": 33
+                }
+              }
+            },
+            {
+              "name": "updateOne",
+              "arguments": {
+                "filter": {
+                  "_id": 2
+                },
+                "update": {
+                  "$inc": {
+                    "x": 1
+                  }
+                }
+              }
+            }
+          ],
+          "options": {
+            "ordered": true
+          }
+        }
+      },
+      "outcome": {
+        "result": {
+          "deletedCount": 1,
+          "insertedIds": {
+            "1": 3
+          },
+          "matchedCount": 1,
+          "modifiedCount": 1,
+          "upsertedCount": 0,
+          "upsertedIds": {}
+        },
+        "collection": {
+          "data": [
+            {
+              "_id": 2,
+              "x": 23
+            },
+            {
+              "_id": 3,
+              "x": 33
+            }
+          ]
+        }
+      }
+    }
+  ]
+}