From a006b595256633ea0b3873a80517f08814d81e29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Llei=CC=88r=20Borra=CC=80s=20Metje?= Date: Sun, 23 Jul 2017 21:59:32 +0200 Subject: [PATCH 01/12] Fix PG::UnableToSend using a reconnect --- lib/ruote/postgres/storage.rb | 17 +++++++++++++++++ spec/support/helpers.rb | 4 ++++ spec/unit/storage_spec.rb | 26 ++++++++++++++++++++++++++ 3 files changed, 47 insertions(+) diff --git a/lib/ruote/postgres/storage.rb b/lib/ruote/postgres/storage.rb index 560bb2b..24a72ec 100644 --- a/lib/ruote/postgres/storage.rb +++ b/lib/ruote/postgres/storage.rb @@ -114,6 +114,7 @@ def initialize(pg, options={}) @table = options.fetch('pg_table_name', :documents).to_sym @abort_on_connection_error = options.fetch('abort_on_connection_error', true) + @retry_on_connection_error = options.fetch('retry_on_connection_error', false) replace_engine_configuration(options) end @@ -297,14 +298,30 @@ def has_json? server_version[0] >= 9 && server_version[1] >= 2 end + def reconnect + @pg = db_connect + end + + def db_connect + PGconn.connect_start(@pg.host, @pg.port, @pg.options, @pg.tty, @pg.db, @pg.user, @pg.pass) + end + def safe_pg(&block) + retries ||= 0 + @mutex.synchronize do yield end rescue *CONNECTION_ERRORS => e + if @retry_on_connection_error && retries < 1 + retries += 1 + reconnect && retry + end + if @abort_on_connection_error abort "ruote-postgres fatal error: #{e.class.name} #{e.message}\n#{e.backtrace.join("\n")}" else + raise e end end diff --git a/spec/support/helpers.rb b/spec/support/helpers.rb index 44c1b88..9bf3ad9 100644 --- a/spec/support/helpers.rb +++ b/spec/support/helpers.rb @@ -7,6 +7,10 @@ def db_connect() PG.connect(config) end + def db_unable_to_send_connect() + PGconn.connect_start( '127.0.0.1', 54320, "", "", "me", "xxxx", "somedb" ) + end + def columns(pg, table) pg.exec(%{SELECT attname FROM pg_attribute, pg_type WHERE typname = '#{table}' diff --git a/spec/unit/storage_spec.rb b/spec/unit/storage_spec.rb index 230964d..9466743 100644 --- a/spec/unit/storage_spec.rb +++ b/spec/unit/storage_spec.rb @@ -43,6 +43,32 @@ end end + describe "connection errors" do + subject do + Ruote::Postgres::Storage.new(pg, storage_options).tap do |rps| + rps.instance_variable_set :@pg, db_unable_to_send_connect + end + end + + context "triggers UnableToSend" do + let(:storage_options) { { 'retry_on_connection_error' => false, 'abort_on_connection_error' => false } } + + it "raises the PG error" do + expect { subject.get('msgs', '1') }.to raise_error(PG::UnableToSend) + end + end + + context "retries on UnableToSend" do + let(:storage_options) { { 'retry_on_connection_error' => true } } + + it "raises the PG error" do + expect(subject).to receive(:db_connect).and_return(db_connect) + + expect { subject.get('msgs', '1') }.not_to raise_error + end + end + end + describe "interface" do subject { Ruote::Postgres::Storage.new(pg) } From 4700167d2671e731560525fd448694a440609b40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Llei=CC=88r=20Borra=CC=80s=20Metje?= Date: Mon, 24 Jul 2017 10:37:38 +0200 Subject: [PATCH 02/12] allow custom number of retries --- lib/ruote/postgres/storage.rb | 19 ++++++++++--------- spec/unit/storage_spec.rb | 20 +++++++++++++++----- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/lib/ruote/postgres/storage.rb b/lib/ruote/postgres/storage.rb index 24a72ec..f43ed74 100644 --- a/lib/ruote/postgres/storage.rb +++ b/lib/ruote/postgres/storage.rb @@ -112,9 +112,9 @@ def initialize(pg, options={}) @mutex = Mutex.new @pg = pg - @table = options.fetch('pg_table_name', :documents).to_sym - @abort_on_connection_error = options.fetch('abort_on_connection_error', true) - @retry_on_connection_error = options.fetch('retry_on_connection_error', false) + @table = options.fetch('pg_table_name', :documents).to_sym + @abort_on_connection_error = options.fetch('abort_on_connection_error', true) + @retries_on_connection_error = options.fetch('retries_on_connection_error', 0).to_i replace_engine_configuration(options) end @@ -313,16 +313,17 @@ def safe_pg(&block) yield end rescue *CONNECTION_ERRORS => e - if @retry_on_connection_error && retries < 1 + result = if retries < @retries_on_connection_error retries += 1 reconnect && retry end - if @abort_on_connection_error - abort "ruote-postgres fatal error: #{e.class.name} #{e.message}\n#{e.backtrace.join("\n")}" - else - - raise e + unless result + if @abort_on_connection_error + abort "ruote-postgres fatal error: #{e.class.name} #{e.message}\n#{e.backtrace.join("\n")}" + else + raise e + end end end end diff --git a/spec/unit/storage_spec.rb b/spec/unit/storage_spec.rb index 9466743..924eb8c 100644 --- a/spec/unit/storage_spec.rb +++ b/spec/unit/storage_spec.rb @@ -45,28 +45,38 @@ describe "connection errors" do subject do - Ruote::Postgres::Storage.new(pg, storage_options).tap do |rps| + Ruote::Postgres::Storage.new(pg, storage_options.merge({'abort_on_connection_error' => false})).tap do |rps| rps.instance_variable_set :@pg, db_unable_to_send_connect end end context "triggers UnableToSend" do - let(:storage_options) { { 'retry_on_connection_error' => false, 'abort_on_connection_error' => false } } + let(:storage_options) { { 'retries_on_connection_error' => 0 } } it "raises the PG error" do expect { subject.get('msgs', '1') }.to raise_error(PG::UnableToSend) end end - context "retries on UnableToSend" do - let(:storage_options) { { 'retry_on_connection_error' => true } } + context "retries on time on UnableToSend" do + let(:storage_options) { { 'retries_on_connection_error' => 1 } } - it "raises the PG error" do + it "rescues the PG error" do expect(subject).to receive(:db_connect).and_return(db_connect) expect { subject.get('msgs', '1') }.not_to raise_error end end + + context "retries many times on UnableToSend" do + let(:storage_options) { { 'retries_on_connection_error' => 10 } } + + it "raises the PG error" do + expect(subject).to receive(:reconnect).exactly(10).times.and_return(db_unable_to_send_connect) + + expect { subject.get('msgs', '1') }.to raise_error(PG::UnableToSend) + end + end end describe "interface" do From ddefe026ab8198c624824bc38748ebb354b0c862 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Llei=CC=88r=20Borra=CC=80s=20Metje?= Date: Mon, 31 Jul 2017 17:36:57 +0200 Subject: [PATCH 03/12] Add some debugging messages --- lib/ruote/postgres/storage.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/ruote/postgres/storage.rb b/lib/ruote/postgres/storage.rb index f43ed74..aae7a93 100644 --- a/lib/ruote/postgres/storage.rb +++ b/lib/ruote/postgres/storage.rb @@ -313,6 +313,8 @@ def safe_pg(&block) yield end rescue *CONNECTION_ERRORS => e + $stderr.puts "CONNECTION ERROR => Retries #{retries}" + $stderr.puts e result = if retries < @retries_on_connection_error retries += 1 reconnect && retry From ee3ee06aab9e132454c71f6ab92d9bc43d1320a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Llei=CC=88r=20Borra=CC=80s=20Metje?= Date: Mon, 31 Jul 2017 17:52:55 +0200 Subject: [PATCH 04/12] Verbosity --- lib/ruote/postgres/storage.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/ruote/postgres/storage.rb b/lib/ruote/postgres/storage.rb index aae7a93..14a877d 100644 --- a/lib/ruote/postgres/storage.rb +++ b/lib/ruote/postgres/storage.rb @@ -109,6 +109,7 @@ class Storage attr_reader :pg def initialize(pg, options={}) + $stderr.puts "INITIALIZE #{pg}" @mutex = Mutex.new @pg = pg @@ -299,10 +300,12 @@ def has_json? end def reconnect + @stderr.puts "RECONNECT" @pg = db_connect end def db_connect + $stderr.puts "DB_CONNECT #{@pg.host}, #{@pg.port}, #{@pg.options}, #{@pg.tty}, #{@pg.db}, #{@pg.user}, #{@pg.pass}" PGconn.connect_start(@pg.host, @pg.port, @pg.options, @pg.tty, @pg.db, @pg.user, @pg.pass) end From 20eaf14fdc9e18e11fd34a39cb1ca740382a8f08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Llei=CC=88r=20Borra=CC=80s=20Metje?= Date: Mon, 7 Aug 2017 13:15:58 +0200 Subject: [PATCH 05/12] Better verboser messaging --- lib/ruote/postgres/storage.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/ruote/postgres/storage.rb b/lib/ruote/postgres/storage.rb index 14a877d..ee0605f 100644 --- a/lib/ruote/postgres/storage.rb +++ b/lib/ruote/postgres/storage.rb @@ -109,7 +109,6 @@ class Storage attr_reader :pg def initialize(pg, options={}) - $stderr.puts "INITIALIZE #{pg}" @mutex = Mutex.new @pg = pg @@ -300,12 +299,12 @@ def has_json? end def reconnect - @stderr.puts "RECONNECT" + @stderr.puts "[RP] RECONNECT" @pg = db_connect end def db_connect - $stderr.puts "DB_CONNECT #{@pg.host}, #{@pg.port}, #{@pg.options}, #{@pg.tty}, #{@pg.db}, #{@pg.user}, #{@pg.pass}" + $stderr.puts "[RP] DB_CONNECT #{@pg.host}, #{@pg.port}, #{@pg.options}, #{@pg.tty}, #{@pg.db}, #{@pg.user}, #{@pg.pass}" PGconn.connect_start(@pg.host, @pg.port, @pg.options, @pg.tty, @pg.db, @pg.user, @pg.pass) end @@ -316,13 +315,14 @@ def safe_pg(&block) yield end rescue *CONNECTION_ERRORS => e - $stderr.puts "CONNECTION ERROR => Retries #{retries}" + $stderr.puts "[RP] CONNECTION_ERROR => Retries #{retries}" $stderr.puts e result = if retries < @retries_on_connection_error retries += 1 reconnect && retry end + $stderr.puts result unless result if @abort_on_connection_error abort "ruote-postgres fatal error: #{e.class.name} #{e.message}\n#{e.backtrace.join("\n")}" From be7a9aa580a879dcc8c15df910f499abac5544fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Llei=CC=88r=20Borra=CC=80s=20Metje?= Date: Mon, 7 Aug 2017 13:21:03 +0200 Subject: [PATCH 06/12] Log timestamps --- lib/ruote/postgres/storage.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/ruote/postgres/storage.rb b/lib/ruote/postgres/storage.rb index ee0605f..d093322 100644 --- a/lib/ruote/postgres/storage.rb +++ b/lib/ruote/postgres/storage.rb @@ -299,12 +299,12 @@ def has_json? end def reconnect - @stderr.puts "[RP] RECONNECT" + @stderr.puts "[RP] #{Time.now} RECONNECT" @pg = db_connect end def db_connect - $stderr.puts "[RP] DB_CONNECT #{@pg.host}, #{@pg.port}, #{@pg.options}, #{@pg.tty}, #{@pg.db}, #{@pg.user}, #{@pg.pass}" + $stderr.puts "[RP] #{Time.now} DB_CONNECT #{@pg.host}, #{@pg.port}, #{@pg.options}, #{@pg.tty}, #{@pg.db}, #{@pg.user}, #{@pg.pass}" PGconn.connect_start(@pg.host, @pg.port, @pg.options, @pg.tty, @pg.db, @pg.user, @pg.pass) end @@ -315,7 +315,7 @@ def safe_pg(&block) yield end rescue *CONNECTION_ERRORS => e - $stderr.puts "[RP] CONNECTION_ERROR => Retries #{retries}" + $stderr.puts "[RP] #{Time.now} CONNECTION_ERROR => Retries #{retries}" $stderr.puts e result = if retries < @retries_on_connection_error retries += 1 From 69b2deba199dea294ed49d0b488a3dee4519389e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Llei=CC=88r=20Borra=CC=80s=20Metje?= Date: Mon, 7 Aug 2017 13:24:52 +0200 Subject: [PATCH 07/12] Better debug messages --- lib/ruote/postgres/storage.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ruote/postgres/storage.rb b/lib/ruote/postgres/storage.rb index d093322..d55c02f 100644 --- a/lib/ruote/postgres/storage.rb +++ b/lib/ruote/postgres/storage.rb @@ -315,7 +315,7 @@ def safe_pg(&block) yield end rescue *CONNECTION_ERRORS => e - $stderr.puts "[RP] #{Time.now} CONNECTION_ERROR => Retries #{retries}" + $stderr.puts "[RP] #{Time.now} CONNECTION_ERROR => Retries #{retries} out of #{@retries_on_connection_error}" $stderr.puts e result = if retries < @retries_on_connection_error retries += 1 From b30445e1673bb98086516399cf4c4700182eee26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Llei=CC=88r=20Borra=CC=80s=20Metje?= Date: Mon, 7 Aug 2017 15:07:56 +0200 Subject: [PATCH 08/12] Fix typo --- lib/ruote/postgres/storage.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ruote/postgres/storage.rb b/lib/ruote/postgres/storage.rb index d55c02f..660974a 100644 --- a/lib/ruote/postgres/storage.rb +++ b/lib/ruote/postgres/storage.rb @@ -299,7 +299,7 @@ def has_json? end def reconnect - @stderr.puts "[RP] #{Time.now} RECONNECT" + $stderr.puts "[RP] #{Time.now} RECONNECT" @pg = db_connect end From aa6f49736a75fa295d41c478b5ebfd67d7e03590 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Llei=CC=88r=20Borra=CC=80s=20Metje?= Date: Thu, 10 Aug 2017 11:35:09 +0200 Subject: [PATCH 09/12] Correctly reconnect to postgres --- lib/ruote/postgres/storage.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ruote/postgres/storage.rb b/lib/ruote/postgres/storage.rb index 660974a..4130166 100644 --- a/lib/ruote/postgres/storage.rb +++ b/lib/ruote/postgres/storage.rb @@ -304,8 +304,8 @@ def reconnect end def db_connect - $stderr.puts "[RP] #{Time.now} DB_CONNECT #{@pg.host}, #{@pg.port}, #{@pg.options}, #{@pg.tty}, #{@pg.db}, #{@pg.user}, #{@pg.pass}" - PGconn.connect_start(@pg.host, @pg.port, @pg.options, @pg.tty, @pg.db, @pg.user, @pg.pass) + $stderr.puts "[RP] #{Time.now} DB_CONNECT host: #{@pg.host}, port: #{@pg.port}, options: #{@pg.options}, tty: #{@pg.tty}, dbname: #{@pg.db}, user: #{@pg.user}, password: #{@pg.pass}" + PG.connect(host: @pg.host, port: @pg.port, dbname: @pg.db, user: @pg.user, password: @pg.pass, options: @pg.options, tty: @pg.tty) end def safe_pg(&block) From 92f244722edd629736e2614af806e236911130a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lle=C3=AFr=20Borr=C3=A0s=20Metje?= Date: Wed, 16 Aug 2017 10:19:16 +0200 Subject: [PATCH 10/12] Do not log the password --- lib/ruote/postgres/storage.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ruote/postgres/storage.rb b/lib/ruote/postgres/storage.rb index 4130166..f529b8d 100644 --- a/lib/ruote/postgres/storage.rb +++ b/lib/ruote/postgres/storage.rb @@ -304,7 +304,7 @@ def reconnect end def db_connect - $stderr.puts "[RP] #{Time.now} DB_CONNECT host: #{@pg.host}, port: #{@pg.port}, options: #{@pg.options}, tty: #{@pg.tty}, dbname: #{@pg.db}, user: #{@pg.user}, password: #{@pg.pass}" + $stderr.puts "[RP] #{Time.now} DB_CONNECT host: #{@pg.host}, port: #{@pg.port}, options: #{@pg.options}, tty: #{@pg.tty}, dbname: #{@pg.db}, user: #{@pg.user}, password: FooBarBaz123" PG.connect(host: @pg.host, port: @pg.port, dbname: @pg.db, user: @pg.user, password: @pg.pass, options: @pg.options, tty: @pg.tty) end From 9c58eb306b73844d4b1b9a1efcfc3d9ee7ad9973 Mon Sep 17 00:00:00 2001 From: Mirko Mignini Date: Tue, 29 Aug 2017 17:38:48 +0200 Subject: [PATCH 11/12] Using reset instead of recreating a connection --- lib/ruote/postgres/storage.rb | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/lib/ruote/postgres/storage.rb b/lib/ruote/postgres/storage.rb index f529b8d..df02073 100644 --- a/lib/ruote/postgres/storage.rb +++ b/lib/ruote/postgres/storage.rb @@ -29,7 +29,6 @@ require 'ruote/storage/base' require 'ruote/postgres/version' - module Ruote module Postgres CONNECTION_ERRORS = [ @@ -300,17 +299,11 @@ def has_json? def reconnect $stderr.puts "[RP] #{Time.now} RECONNECT" - @pg = db_connect - end - - def db_connect - $stderr.puts "[RP] #{Time.now} DB_CONNECT host: #{@pg.host}, port: #{@pg.port}, options: #{@pg.options}, tty: #{@pg.tty}, dbname: #{@pg.db}, user: #{@pg.user}, password: FooBarBaz123" - PG.connect(host: @pg.host, port: @pg.port, dbname: @pg.db, user: @pg.user, password: @pg.pass, options: @pg.options, tty: @pg.tty) + @pg.reset end def safe_pg(&block) retries ||= 0 - @mutex.synchronize do yield end From ce209dcb0af9022b513172748d96d76d16e2677b Mon Sep 17 00:00:00 2001 From: Mirko Mignini Date: Fri, 15 Sep 2017 11:56:09 +0200 Subject: [PATCH 12/12] Add a sleep between every retry --- lib/ruote/postgres/storage.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/ruote/postgres/storage.rb b/lib/ruote/postgres/storage.rb index df02073..e2ab82f 100644 --- a/lib/ruote/postgres/storage.rb +++ b/lib/ruote/postgres/storage.rb @@ -312,6 +312,7 @@ def safe_pg(&block) $stderr.puts e result = if retries < @retries_on_connection_error retries += 1 + sleep 0.5 reconnect && retry end