From 61b507f8ce107221604ebca0c6c4058cfab650a7 Mon Sep 17 00:00:00 2001 From: Kevin Lang Date: Thu, 18 Mar 2021 12:08:44 -0400 Subject: [PATCH 1/3] add fix for thundering herd on initial connection pool setup #8 --- lib/ecto/adapters/sqlite3/connection.ex | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/lib/ecto/adapters/sqlite3/connection.ex b/lib/ecto/adapters/sqlite3/connection.ex index 59dcbd9..36f0241 100644 --- a/lib/ecto/adapters/sqlite3/connection.ex +++ b/lib/ecto/adapters/sqlite3/connection.ex @@ -13,10 +13,20 @@ defmodule Ecto.Adapters.SQLite3.Connection do import Ecto.Adapters.SQLite3.DataType @parent_as __MODULE__ - @connect_buffer 50 + @idx_connect_buffer 5 + @rand_connect_buffer 50 + + # db_connection eagerly obtains and holds the configured + # pool_size of connections, which can cause us to hit + # "SQLite Busy" issues. we solve this by making sure to wait + # a bit by hooking into the :configure callback opt and + # sleeping an amount of time which is a function of the pool_index + def handle_thundering_herd(opts) do + case Keyword.get(opts, :pool_index) do + idx -> :timer.sleep(idx * @idx_connect_buffer) + nil -> :timer.sleep(:rand.uniform(@connect_buffer)) + end - def sleep(opts) do - :timer.sleep(:rand.uniform(@connect_buffer)) opts end @@ -24,7 +34,7 @@ defmodule Ecto.Adapters.SQLite3.Connection do # todo: we may want to consider wrapping any provided :configure # with our custom connection buffering logic opts - |> Keyword.put_new(:configure, {__MODULE__, :sleep, []}) + |> Keyword.put_new(:configure, &handle_thundering_herd/1) end def start_link(opts) do From 0fdb5598d8fd837622f2224b742321656a4da59e Mon Sep 17 00:00:00 2001 From: Kevin Lang Date: Thu, 18 Mar 2021 12:20:40 -0400 Subject: [PATCH 2/3] rand_ --- lib/ecto/adapters/sqlite3/connection.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ecto/adapters/sqlite3/connection.ex b/lib/ecto/adapters/sqlite3/connection.ex index 36f0241..fb37acf 100644 --- a/lib/ecto/adapters/sqlite3/connection.ex +++ b/lib/ecto/adapters/sqlite3/connection.ex @@ -24,7 +24,7 @@ defmodule Ecto.Adapters.SQLite3.Connection do def handle_thundering_herd(opts) do case Keyword.get(opts, :pool_index) do idx -> :timer.sleep(idx * @idx_connect_buffer) - nil -> :timer.sleep(:rand.uniform(@connect_buffer)) + nil -> :timer.sleep(:rand.uniform(@rand_connect_buffer)) end opts From cecdc32535be505c3cf0a6d73b3c40312a87cdbf Mon Sep 17 00:00:00 2001 From: Kevin Lang Date: Thu, 18 Mar 2021 12:47:41 -0400 Subject: [PATCH 3/3] better match --- lib/ecto/adapters/sqlite3/connection.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ecto/adapters/sqlite3/connection.ex b/lib/ecto/adapters/sqlite3/connection.ex index fb37acf..0d5548c 100644 --- a/lib/ecto/adapters/sqlite3/connection.ex +++ b/lib/ecto/adapters/sqlite3/connection.ex @@ -23,7 +23,7 @@ defmodule Ecto.Adapters.SQLite3.Connection do # sleeping an amount of time which is a function of the pool_index def handle_thundering_herd(opts) do case Keyword.get(opts, :pool_index) do - idx -> :timer.sleep(idx * @idx_connect_buffer) + idx when is_integer(idx) -> :timer.sleep(idx * @idx_connect_buffer) nil -> :timer.sleep(:rand.uniform(@rand_connect_buffer)) end