Automatically reconnecting to the database when safe to do so

This has been anf often requested feature for the database layer, specially by people using the database in long running background tasks. Instead of instructing them to figure out how to configure their database to not drop connections, better just give them an extra hand by reconnecting to the database whenever it is safe to do so. That is, reconnection are attempted only on fresh commands to the database outside of a transaction.
cakephp · Jan 28, 2018 · 7d07e3d · 7d07e3d
1 parent 7d75c95
commit 7d07e3d
Show file tree

Hide file tree

Showing 6 changed files with 489 additions and 44 deletions.
diff --git a/src/Core/Retry/CommandRetry.php b/src/Core/Retry/CommandRetry.php
@@ -0,0 +1,80 @@
+<?php
+/**
+ * CakePHP(tm) : Rapid Development Framework (https://cakephp.org)
+ * Copyright (c) Cake Software Foundation, Inc. (https://cakefoundation.org)
+ *
+ * Licensed under The MIT License
+ * For full copyright and license information, please see the LICENSE.txt
+ * Redistributions of files must retain the above copyright notice.
+ *
+ * @copyright     Copyright (c) Cake Software Foundation, Inc. (https://cakefoundation.org)
+ * @link          https://cakephp.org CakePHP(tm) Project
+ * @since         3.6.0
+ * @license       https://opensource.org/licenses/mit-license.php MIT License
+ */
+namespace Cake\Core\Retry;
+
+use Exception;
+
+/**
+ * Allows any action to be retried in case of an exception.
+ *
+ * This class can be parametrized with an strategy, which will be followed
+ * to determine whether or not the action should be retried.
+ */
+class CommandRetry
+{
+
+    /**
+     * The strategy to follow should the executed action fail
+     *
+     * @var \Cake\Core\Retry\RetryStrategyInterface
+     */
+    protected $strategy;
+
+    /**
+     * The number of retries to perform in case of failure
+     *
+     * @var int
+     */
+    protected $retries;
+
+    /**
+     * Creates the CommandRetry object with the given strategy and retry count
+     *
+     * @param \Cake\Core\Retry\RetryStrategyInterface $strategy The strategy to follow should the action fail
+     * @param int $retries The number of times the action has been already called
+     */
+    public function __construct(RetryStrategyInterface $strategy, $retries = 1)
+    {
+        $this->strategy = $strategy;
+        $this->retries = $retries;
+    }
+
+    /**
+     * The number of retries to perform in case of failure
+     *
+     * @param callable $action The callable action to execute with a retry strategy
+     * @return mixed The return value of the passed action callable
+     */
+    public function run(callable $action)
+    {
+        $retryCount = 0;
+        $lastException = null;
+        while ($this->retries > $retryCount) {
+            $retryCount++;
+            try {
+                return $action();
+            } catch (Exception $e) {
+                $lastException = $e;
+                if (!$this->strategy->shouldRetry($e, $retryCount)) {
+                    throw $e;
+                }
+            }
+        }
+
+        if ($lastException !== null) {
+            throw $lastException;
+        }
+    }
+}
diff --git a/src/Core/Retry/RetryStrategyInterface.php b/src/Core/Retry/RetryStrategyInterface.php
@@ -0,0 +1,33 @@
+<?php
+/**
+ * CakePHP(tm) : Rapid Development Framework (https://cakephp.org)
+ * Copyright (c) Cake Software Foundation, Inc. (https://cakefoundation.org)
+ *
+ * Licensed under The MIT License
+ * For full copyright and license information, please see the LICENSE.txt
+ * Redistributions of files must retain the above copyright notice.
+ *
+ * @copyright     Copyright (c) Cake Software Foundation, Inc. (https://cakefoundation.org)
+ * @link          https://cakephp.org CakePHP(tm) Project
+ * @since         3.6.0
+ * @license       https://opensource.org/licenses/mit-license.php MIT License
+ */
+namespace Cake\Core\Retry;
+
+use Exception;
+
+/**
+ * Makes sure the connection to the database is alive before authorizing
+ * the retry of an action.
+ */
+interface RetryStrategyInterface
+{
+    /**
+     * Returns true if the action can be retried, false otherwise.
+     *
+     * @param Exception $exception The exception that caused the action to fail
+     * @param int $retryCount The number of times the action has been already called
+     * @return bool Whether or not it is OK to retry the action
+     */
+    public function shouldRetry(Exception $exception, $retryCount);
+}
diff --git a/src/Database/Connection.php b/src/Database/Connection.php
@@ -15,13 +15,15 @@
 namespace Cake\Database;
 
 use Cake\Core\App;
+use Cake\Core\Retry\CommandRetry;
 use Cake\Database\Exception\MissingConnectionException;
 use Cake\Database\Exception\MissingDriverException;
 use Cake\Database\Exception\MissingExtensionException;
 use Cake\Database\Exception\NestedTransactionRollbackException;
 use Cake\Database\Log\LoggedQuery;
 use Cake\Database\Log\LoggingStatement;
 use Cake\Database\Log\QueryLogger;
+use Cake\Database\Retry\ReconnectStrategy;
 use Cake\Database\Schema\CachedCollection;
 use Cake\Database\Schema\Collection as SchemaCollection;
 use Cake\Datasource\ConnectionInterface;
@@ -182,6 +184,17 @@ public function setDriver($driver, $config = [])
         return $this;
     }
 
+    /**
+     * Get the retry wrapper object, that is used to recover from server disconnects
+     * while performing ceratain database actions, such as executing a query
+     *
+     * @return \Cake\Core\Retry\CommandRetry The retry wrapper
+     */
+    public function getDisconnectRetry()
+    {
+        return new CommandRetry(new ReconnectStrategy($this));
+    }
+
     /**
      * Gets the driver instance.
      *
@@ -258,13 +271,15 @@ public function isConnected()
      */
     public function prepare($sql)
     {
-        $statement = $this->_driver->prepare($sql);
+        return $this->getDisconnectRetry()->run(function () use ($sql) {
+            $statement = $this->_driver->prepare($sql);
 
-        if ($this->_logQueries) {
-            $statement = $this->_newLogger($statement);
-        }
+            if ($this->_logQueries) {
+                $statement = $this->_newLogger($statement);
+            }
 
-        return $statement;
+            return $statement;
+        });
     }
 
     /**
@@ -278,15 +293,17 @@ public function prepare($sql)
      */
     public function execute($query, array $params = [], array $types = [])
     {
-        if (!empty($params)) {
-            $statement = $this->prepare($query);
-            $statement->bind($params, $types);
-            $statement->execute();
-        } else {
-            $statement = $this->query($query);
-        }
+        return $this->getDisconnectRetry()->run(function () use ($query, $params, $types) {
+            if (!empty($params)) {
+                $statement = $this->prepare($query);
+                $statement->bind($params, $types);
+                $statement->execute();
+            } else {
+                $statement = $this->query($query);
+            }
 
-        return $statement;
+            return $statement;
+        });
     }
 
     /**
@@ -311,11 +328,13 @@ public function compileQuery(Query $query, ValueBinder $generator)
      */
     public function run(Query $query)
     {
-        $statement = $this->prepare($query);
-        $query->getValueBinder()->attachTo($statement);
-        $statement->execute();
+        return $this->getDisconnectRetry()->run(function () use ($query) {
+            $statement = $this->prepare($query);
+            $query->getValueBinder()->attachTo($statement);
+            $statement->execute();
 
-        return $statement;
+            return $statement;
+        });
     }
 
     /**
@@ -326,10 +345,12 @@ public function run(Query $query)
      */
     public function query($sql)
     {
-        $statement = $this->prepare($sql);
-        $statement->execute();
+        return $this->getDisconnectRetry()->run(function () use ($sql) {
+            $statement = $this->prepare($sql);
+            $statement->execute();
 
-        return $statement;
+            return $statement;
+        });
     }
 
     /**
@@ -403,12 +424,14 @@ public function schemaCollection(SchemaCollection $collection = null)
      */
     public function insert($table, array $data, array $types = [])
     {
-        $columns = array_keys($data);
+        return $this->getDisconnectRetry()->run(function () use ($table, $data, $types) {
+            $columns = array_keys($data);
 
-        return $this->newQuery()->insert($columns, $types)
-            ->into($table)
-            ->values($data)
-            ->execute();
+            return $this->newQuery()->insert($columns, $types)
+                ->into($table)
+                ->values($data)
+                ->execute();
+        });
     }
 
     /**
@@ -422,10 +445,12 @@ public function insert($table, array $data, array $types = [])
      */
     public function update($table, array $data, array $conditions = [], $types = [])
     {
-        return $this->newQuery()->update($table)
-            ->set($data, $types)
-            ->where($conditions, $types)
-            ->execute();
+        return $this->getDisconnectRetry()->run(function () use ($table, $data, $conditions, $types) {
+            return $this->newQuery()->update($table)
+                ->set($data, $types)
+                ->where($conditions, $types)
+                ->execute();
+        });
     }
 
     /**
@@ -438,9 +463,11 @@ public function update($table, array $data, array $conditions = [], $types = [])
      */
     public function delete($table, $conditions = [], $types = [])
     {
-        return $this->newQuery()->delete($table)
-            ->where($conditions, $types)
-            ->execute();
+        return $this->getDisconnectRetry()->run(function () use ($table, $conditions, $types) {
+            return $this->newQuery()->delete($table)
+                ->where($conditions, $types)
+                ->execute();
+        });
     }
 
     /**
@@ -454,7 +481,11 @@ public function begin()
             if ($this->_logQueries) {
                 $this->log('BEGIN');
             }
-            $this->_driver->beginTransaction();
+
+            $this->getDisconnectRetry()->run(function () {
+                $this->_driver->beginTransaction();
+            });
+
             $this->_transactionLevel = 0;
             $this->_transactionStarted = true;
             $this->nestedTransactionRollbackException = null;
@@ -648,7 +679,9 @@ public function rollbackSavepoint($name)
      */
     public function disableForeignKeys()
     {
-        $this->execute($this->_driver->disableForeignKeySQL())->closeCursor();
+        $this->getDisconnectRetry()->run(function () {
+            $this->execute($this->_driver->disableForeignKeySQL())->closeCursor();
+        });
     }
 
     /**
@@ -658,7 +691,9 @@ public function disableForeignKeys()
      */
     public function enableForeignKeys()
     {
-        $this->execute($this->_driver->enableForeignKeySQL())->closeCursor();
+        $this->getDisconnectRetry()->run(function () {
+            $this->execute($this->_driver->enableForeignKeySQL())->closeCursor();
+        });
     }
 
     /**
@@ -733,18 +768,20 @@ protected function wasNestedTransactionRolledback()
      */
     public function disableConstraints(callable $callback)
     {
-        $this->disableForeignKeys();
+        return $this->getDisconnectRetry()->run(function () use ($callback) {
+            $this->disableForeignKeys();
 
-        try {
-            $result = $callback($this);
-        } catch (Exception $e) {
-            $this->enableForeignKeys();
-            throw $e;
-        }
+            try {
+                $result = $callback($this);
+            } catch (Exception $e) {
+                $this->enableForeignKeys();
+                throw $e;
+            }
 
-        $this->enableForeignKeys();
+            $this->enableForeignKeys();
 
-        return $result;
+            return $result;
+        });
     }
 
     /**