From 87fb2880a36593bb257766347ff7009b5c538921 Mon Sep 17 00:00:00 2001 From: Kevin Lang Date: Thu, 18 Mar 2021 15:40:36 -0400 Subject: [PATCH 1/3] disable doc for internal modules --- lib/ecto/adapters/sqlite3/codec.ex | 2 ++ lib/ecto/adapters/sqlite3/connection.ex | 2 ++ lib/ecto/adapters/sqlite3/data_type.ex | 2 ++ 3 files changed, 6 insertions(+) diff --git a/lib/ecto/adapters/sqlite3/codec.ex b/lib/ecto/adapters/sqlite3/codec.ex index 1a6678e..e267c01 100644 --- a/lib/ecto/adapters/sqlite3/codec.ex +++ b/lib/ecto/adapters/sqlite3/codec.ex @@ -1,4 +1,6 @@ defmodule Ecto.Adapters.SQLite3.Codec do + @moduledoc false + def bool_decode(nil), do: {:ok, nil} def bool_decode(0), do: {:ok, false} def bool_decode("0"), do: {:ok, false} diff --git a/lib/ecto/adapters/sqlite3/connection.ex b/lib/ecto/adapters/sqlite3/connection.ex index 0d5548c..97eacff 100644 --- a/lib/ecto/adapters/sqlite3/connection.ex +++ b/lib/ecto/adapters/sqlite3/connection.ex @@ -1,4 +1,6 @@ defmodule Ecto.Adapters.SQLite3.Connection do + @moduledoc false + @behaviour Ecto.Adapters.SQL.Connection alias Ecto.Migration.Constraint diff --git a/lib/ecto/adapters/sqlite3/data_type.ex b/lib/ecto/adapters/sqlite3/data_type.ex index c8127f9..2159e6b 100644 --- a/lib/ecto/adapters/sqlite3/data_type.ex +++ b/lib/ecto/adapters/sqlite3/data_type.ex @@ -1,4 +1,6 @@ defmodule Ecto.Adapters.SQLite3.DataType do + @moduledoc false + # Simple column types. Note that we ignore options like :size, :precision, # etc. because columns do not have types, and SQLite will not coerce any # stored value. Thus, "strings" are all text and "numerics" have arbitrary From 91466e2007ebf56f8f92a40821c126c789e8130c Mon Sep 17 00:00:00 2001 From: Kevin Lang Date: Thu, 18 Mar 2021 20:53:31 -0400 Subject: [PATCH 2/3] update defaults, add documentation --- integration_test/test_helper.exs | 8 -- lib/ecto/adapters/sqlite3.ex | 103 ++++++++++++++++++++++++ lib/ecto/adapters/sqlite3/connection.ex | 4 + mix.exs | 12 ++- test/test_helper.exs | 4 - 5 files changed, 118 insertions(+), 13 deletions(-) diff --git a/integration_test/test_helper.exs b/integration_test/test_helper.exs index c2b8a8a..1268091 100644 --- a/integration_test/test_helper.exs +++ b/integration_test/test_helper.exs @@ -13,11 +13,7 @@ alias Ecto.Integration.TestRepo Application.put_env(:ecto_sqlite3, TestRepo, adapter: Ecto.Adapters.SQLite3, database: "/tmp/exqlite_integration_test.db", - journal_mode: :wal, - cache_size: -64000, - temp_store: :memory, pool: Ecto.Adapters.SQL.Sandbox, - pool_size: 5, show_sensitive_data_on_connection_error: true ) @@ -27,10 +23,6 @@ alias Ecto.Integration.PoolRepo Application.put_env(:ecto_sqlite3, PoolRepo, adapter: Ecto.Adapters.SQLite3, database: "/tmp/exqlite_integration_pool_test.db", - journal_mode: :wal, - cache_size: -64000, - temp_store: :memory, - pool_size: 5, show_sensitive_data_on_connection_error: true ) diff --git a/lib/ecto/adapters/sqlite3.ex b/lib/ecto/adapters/sqlite3.ex index c80a3aa..dde70ce 100644 --- a/lib/ecto/adapters/sqlite3.ex +++ b/lib/ecto/adapters/sqlite3.ex @@ -1,4 +1,107 @@ defmodule Ecto.Adapters.SQLite3 do + @moduledoc """ + Adapter module for SQLite3. + + It uses `Exqlite` for communicating to the database. + + ## Options + + The adapter supports a superset of the options provided by the + underlying `Exqlite` driver. + + ### Provided options + + * `:database` - The path to the database. In memory is allowed. You can use + `:memory` or `":memory:"` to designate that. + * `:journal_mode` - Sets the journal mode for the sqlite connection. Can be + one of the following `:delete`, `:truncate`, `:persist`, `:memory`, + `:wal`, or `:off`. Defaults to `:wal`. + * `:temp_store` - Sets the storage used for temporary tables. Default is + `:default`. Allowed values are `:default`, `:file`, `:memory`. + * `:synchronous` - Can be `:extra`, `:full`, `:normal`, or `:off`. Defaults + to `:normal`. + * `:foreign_keys` - Sets if foreign key checks should be enforced or not. + Can be `:on` or `:off`. Default is `:on`. + * `:cache_size` - Sets the cache size to be used for the connection. This is + an odd setting as a positive value is the number of pages in memory to use + and a negative value is the size in kilobytes to use. Default is `-64000`. + * `:cache_spill` - The cache_spill pragma enables or disables the ability of + the pager to spill dirty cache pages to the database file in the middle of + a transaction. By default it is `:on`, and for most applications, it + should remain so. + * `:case_sensitive_like` - whether LIKE is case-sensitive or not. Can be + `:off` or `:on`. Defaults to `:off`. + * `:auto_vacuum` - Defaults to `:none`. Can be `:none`, `:full` or + `:incremental`. Depending on the database size, `:incremental` may be + beneficial. + * `:locking_mode` - Defaults to `:normal`. Allowed values are `:normal` or + `:exclusive`. See [sqlite documenation][1] for more information. + * `:secure_delete` - Defaults to `:off`. Can be `:off` or `:on`. If `:on`, it will cause SQLite3 + to overwrite records that were deleted with zeros. + * `:wal_auto_check_point` - Sets the write-ahead log auto-checkpoint + interval. Default is `1000`. Setting the auto-checkpoint size to zero or a + negative value turns auto-checkpointing off. + * `:busy_timeout` - Sets the busy timeout in milliseconds for a connection. + Default is `2000`. + + For more information about the options above, see [sqlite documenation][1] + + ### Differences between SQLite and Ecto SQLite defaults + + For the most part, the defaults we provide above match the defaults that SQLite usually + ships with. However, SQLite has conservative defaults due to its need to be strictly backwards + compatible, so some of them do not necessarily match "best practices". Below are the defaults + we provide above that differ from the normal SQLite defaults, along with rationale. + + * `:journal_mode` - we use `:wal`, as it is vastly superior + for concurrent access. SQLite usually defaults to `:delete`. + See [SQLite documentation][2] for more info. + * `:temp_store` - we use `:memory`, which increases performance a bit. + SQLite usually defaults to `:file`. + * `:foreign_keys` - we set it to `:on`, for better relational guarantees. + This is also the default of the underlying `Exqlite` driver. + SQLite usually defaults to `:off` for backwards compat. + * `:busy_timeout` - we set it to `2000`, to better enable concurrent access. + This is also the default of `Exqlite`. SQLite usually defaults to `0`. + * `:cache_size` - we set it to `-64000`, to speed up access of data. + SQLite usually defaults to `-2000`. + + These defaults can of course be overridden, as noted above, to suit other needs. + + [1]: https://www.sqlite.org/pragma.html + [2]: https://sqlite.org/wal.html + + ## Limitations + + There are some limitations when using Ecto with SQLite that one needs + to be aware of. The ones listed below are specific to Ecto usage, but it + is encouraged to also view the guidance on [when to use SQLite][4] provided + by the SQLite documentation, as well. + + ### Async Sandbox testing + + The Ecto SQLite3 adapter does not support async tests when used with + `Ecto.Adapters.SQL.Sandbox`. This is due to SQLite only allowing up one + write transaction at a time, which often does not work with the Sandbox approach of + wrapping each test in a transaction. + + ### LIKE match on BLOB columns + + We have the DSQLITE_LIKE_DOESNT_MATCH_BLOBS compile-time option set to true, + as [recommended][3] by SQLite. This means you cannot do LIKE queries on BLOB columns. + + ### Schemaless queries + + Using [schemaless Ecto queries][5] will not work well with SQLite. This is because + the Ecto SQLite adapter relies heavily on the schema to support a rich array of Elixir + types, despite the fact SQLite only has [five storage classes][5]. The query will still + work and return data, but you will need to do this mapping on your own. + + [3]: https://www.sqlite.org/compile.html + [4]: https://www.sqlite.org/whentouse.html + [5]: https://www.sqlite.org/datatype3.html + """ + use Ecto.Adapters.SQL, driver: :exqlite diff --git a/lib/ecto/adapters/sqlite3/connection.ex b/lib/ecto/adapters/sqlite3/connection.ex index 97eacff..bf938fb 100644 --- a/lib/ecto/adapters/sqlite3/connection.ex +++ b/lib/ecto/adapters/sqlite3/connection.ex @@ -36,6 +36,10 @@ defmodule Ecto.Adapters.SQLite3.Connection do # todo: we may want to consider wrapping any provided :configure # with our custom connection buffering logic opts + |> Keyword.put_new(:journal_mode, :wal) + |> Keyword.put_new(:cache_size, -64000) + |> Keyword.put_new(:temp_store, :memory) + |> Keyword.put_new(:pool_size, 5) |> Keyword.put_new(:configure, &handle_thundering_herd/1) end diff --git a/mix.exs b/mix.exs index 8b1f1ef..17b0364 100644 --- a/mix.exs +++ b/mix.exs @@ -13,7 +13,11 @@ defmodule EctoSQLite3.MixProject do package: package(), description: description(), test_paths: test_paths(System.get_env("EXQLITE_INTEGRATION")), - elixirc_paths: elixirc_paths(Mix.env()) + elixirc_paths: elixirc_paths(Mix.env()), + + # Docs + name: "Ecto SQLite3", + docs: docs() ] end @@ -65,6 +69,12 @@ defmodule EctoSQLite3.MixProject do ] end + defp docs do + [ + main: "Ecto.Adapters.SQLite3", + ] + end + defp elixirc_paths(:test), do: ["lib", "test/support"] defp elixirc_paths(_), do: ["lib"] diff --git a/test/test_helper.exs b/test/test_helper.exs index dd5b030..3a84a78 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -11,11 +11,7 @@ alias Ecto.Integration.TestRepo Application.put_env(:ecto_sqlite3, TestRepo, adapter: Ecto.Adapters.SQLite3, database: "/tmp/exqlite_sandbox_test.db", - journal_mode: :wal, - cache_size: -64000, - temp_store: :memory, pool: Ecto.Adapters.SQL.Sandbox, - pool_size: 5, show_sensitive_data_on_connection_error: true ) From 69a0c949136469c63665eed9a2d93e25a1e612b0 Mon Sep 17 00:00:00 2001 From: Kevin Lang Date: Thu, 18 Mar 2021 20:55:31 -0400 Subject: [PATCH 3/3] add doc about pool_size --- lib/ecto/adapters/sqlite3.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/ecto/adapters/sqlite3.ex b/lib/ecto/adapters/sqlite3.ex index dde70ce..90f0680 100644 --- a/lib/ecto/adapters/sqlite3.ex +++ b/lib/ecto/adapters/sqlite3.ex @@ -43,6 +43,7 @@ defmodule Ecto.Adapters.SQLite3 do negative value turns auto-checkpointing off. * `:busy_timeout` - Sets the busy timeout in milliseconds for a connection. Default is `2000`. + * `:pool_size` - the size of the connection pool. Defaults to `5`. For more information about the options above, see [sqlite documenation][1]