From 8ac2af29ad7dc8c86bfb569e3375d8c1cd58a323 Mon Sep 17 00:00:00 2001 From: Stephen Margheim Date: Tue, 2 Jan 2024 12:52:54 +0100 Subject: [PATCH] Allow overriding SQLite defaults from `database.yml` (#50460) * Allow overriding SQLite defaults from `database.yml` Any PRAGMA configuration set under the `pragmas` key in the configuration file take precedence over Rails' defaults, and additional PRAGMAs can be set as well. ```yaml database: storage/development.sqlite3 timeout: 5000 pragmas: synchronous: full temp_store: memory ``` * Style * Allow overriding SQLite defaults from `database.yml` Any PRAGMA configuration set under the `pragmas` key in the configuration file take precedence over Rails' defaults, and additional PRAGMAs can be set as well. ```yaml database: storage/development.sqlite3 timeout: 5000 pragmas: synchronous: full temp_store: memory ``` --------- Co-authored-by: David Heinemeier Hansson --- activerecord/CHANGELOG.md | 14 + .../connection_adapters/sqlite3_adapter.rb | 38 ++- .../adapters/sqlite3/sqlite3_adapter_test.rb | 253 +++++++++++++++++- 3 files changed, 281 insertions(+), 24 deletions(-) diff --git a/activerecord/CHANGELOG.md b/activerecord/CHANGELOG.md index ad782379820f..1ef981e79dd1 100644 --- a/activerecord/CHANGELOG.md +++ b/activerecord/CHANGELOG.md @@ -1,3 +1,17 @@ +* Allow overriding SQLite defaults from `database.yml` + + Any PRAGMA configuration set under the `pragmas` key in the configuration file take precedence over Rails' defaults, and additional PRAGMAs can be set as well. + + ```yaml + database: storage/development.sqlite3 + timeout: 5000 + pragmas: + journal_mode: off + temp_store: memory + ``` + + *Stephen Margheim* + * Remove warning message when running SQLite in production, but leave it unconfigured There are valid use cases for running SQLite in production, however it must be done diff --git a/activerecord/lib/active_record/connection_adapters/sqlite3_adapter.rb b/activerecord/lib/active_record/connection_adapters/sqlite3_adapter.rb index 7445a272e1cf..cc81a5efb8de 100644 --- a/activerecord/lib/active_record/connection_adapters/sqlite3_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/sqlite3_adapter.rb @@ -78,6 +78,15 @@ def dbconsole(config, options = {}) json: { name: "json" }, } + DEFAULT_PRAGMAS = { + "foreign_keys" => true, + "journal_mode" => :wal, + "synchronous" => :normal, + "mmap_size" => 134217728, # 128 megabytes + "journal_size_limit" => 67108864, # 64 megabytes + "cache_size" => 2000 + } + class StatementPool < ConnectionAdapters::StatementPool # :nodoc: alias reset clear @@ -759,29 +768,14 @@ def configure_connection super - # Enforce foreign key constraints - # https://www.sqlite.org/pragma.html#pragma_foreign_keys - # https://www.sqlite.org/foreignkeys.html - raw_execute("PRAGMA foreign_keys = ON", "SCHEMA") - unless @memory_database - # Journal mode WAL allows for greater concurrency (many readers + one writer) - # https://www.sqlite.org/pragma.html#pragma_journal_mode - raw_execute("PRAGMA journal_mode = WAL", "SCHEMA") - # Set more relaxed level of database durability - # 2 = "FULL" (sync on every write), 1 = "NORMAL" (sync every 1000 written pages) and 0 = "NONE" - # https://www.sqlite.org/pragma.html#pragma_synchronous - raw_execute("PRAGMA synchronous = NORMAL", "SCHEMA") - # Set the global memory map so all processes can share some data - # https://www.sqlite.org/pragma.html#pragma_mmap_size - # https://www.sqlite.org/mmap.html - raw_execute("PRAGMA mmap_size = #{128.megabytes}", "SCHEMA") + pragmas = @config.fetch(:pragmas, {}).stringify_keys + DEFAULT_PRAGMAS.merge(pragmas).each do |pragma, value| + if ::SQLite3::Pragmas.method_defined?("#{pragma}=") + @raw_connection.public_send("#{pragma}=", value) + else + warn "Unknown SQLite pragma: #{pragma}" + end end - # Impose a limit on the WAL file to prevent unlimited growth - # https://www.sqlite.org/pragma.html#pragma_journal_size_limit - raw_execute("PRAGMA journal_size_limit = #{64.megabytes}", "SCHEMA") - # Set the local connection cache to 2000 pages - # https://www.sqlite.org/pragma.html#pragma_cache_size - raw_execute("PRAGMA cache_size = 2000", "SCHEMA") end end ActiveSupport.run_load_hooks(:active_record_sqlite3adapter, SQLite3Adapter) diff --git a/activerecord/test/cases/adapters/sqlite3/sqlite3_adapter_test.rb b/activerecord/test/cases/adapters/sqlite3/sqlite3_adapter_test.rb index d9daf9da4980..a3324109e42c 100644 --- a/activerecord/test/cases/adapters/sqlite3/sqlite3_adapter_test.rb +++ b/activerecord/test/cases/adapters/sqlite3/sqlite3_adapter_test.rb @@ -159,7 +159,7 @@ def test_default_pragmas if in_memory_db? assert_equal [{ "foreign_keys" => 1 }], @conn.execute("PRAGMA foreign_keys") assert_equal [{ "journal_mode" => "memory" }], @conn.execute("PRAGMA journal_mode") - assert_equal [{ "synchronous" => 2 }], @conn.execute("PRAGMA synchronous") + assert_equal [{ "synchronous" => 1 }], @conn.execute("PRAGMA synchronous") assert_equal [{ "journal_size_limit" => 67108864 }], @conn.execute("PRAGMA journal_size_limit") assert_equal [], @conn.execute("PRAGMA mmap_size") assert_equal [{ "cache_size" => 2000 }], @conn.execute("PRAGMA cache_size") @@ -175,6 +175,245 @@ def test_default_pragmas end end + def test_overriding_default_foreign_keys_pragma + method_name = in_memory_db? ? :with_memory_connection : :with_file_connection + + send(method_name, pragmas: { foreign_keys: false }) do |conn| + assert_equal [{ "foreign_keys" => 0 }], conn.execute("PRAGMA foreign_keys") + end + + send(method_name, pragmas: { foreign_keys: 0 }) do |conn| + assert_equal [{ "foreign_keys" => 0 }], conn.execute("PRAGMA foreign_keys") + end + + send(method_name, pragmas: { foreign_keys: "false" }) do |conn| + assert_equal [{ "foreign_keys" => 0 }], conn.execute("PRAGMA foreign_keys") + end + + error = assert_raises(ActiveRecord::StatementInvalid) do + send(method_name, pragmas: { foreign_keys: :false }) do |conn| + conn.execute("PRAGMA foreign_keys") + end + end + assert_match(/unrecognized pragma parameter :false/, error.message) + end + + def test_overriding_default_journal_mode_pragma + # in-memory databases are always only ever in `memory` journal_mode + if in_memory_db? + with_memory_connection(pragmas: { "journal_mode" => "delete" }) do |conn| + assert_equal [{ "journal_mode" => "memory" }], conn.execute("PRAGMA journal_mode") + end + + with_memory_connection(pragmas: { "journal_mode" => :delete }) do |conn| + assert_equal [{ "journal_mode" => "memory" }], conn.execute("PRAGMA journal_mode") + end + + error = assert_raises(ActiveRecord::StatementInvalid) do + with_memory_connection(pragmas: { "journal_mode" => 0 }) do |conn| + conn.execute("PRAGMA journal_mode") + end + end + assert_match(/nrecognized journal_mode 0/, error.message) + + error = assert_raises(ActiveRecord::StatementInvalid) do + with_memory_connection(pragmas: { "journal_mode" => false }) do |conn| + conn.execute("PRAGMA journal_mode") + end + end + assert_match(/nrecognized journal_mode false/, error.message) + else + # must use a new, separate database file that hasn't been opened in WAL mode before + with_file_connection(database: "fixtures/journal_mode_test.sqlite3", pragmas: { "journal_mode" => "delete" }) do |conn| + assert_equal [{ "journal_mode" => "delete" }], conn.execute("PRAGMA journal_mode") + end + + with_file_connection(database: "fixtures/journal_mode_test.sqlite3", pragmas: { "journal_mode" => :delete }) do |conn| + assert_equal [{ "journal_mode" => "delete" }], conn.execute("PRAGMA journal_mode") + end + + error = assert_raises(ActiveRecord::StatementInvalid) do + with_file_connection(database: "fixtures/journal_mode_test.sqlite3", pragmas: { "journal_mode" => 0 }) do |conn| + conn.execute("PRAGMA journal_mode") + end + end + assert_match(/unrecognized journal_mode 0/, error.message) + + error = assert_raises(ActiveRecord::StatementInvalid) do + with_file_connection(database: "fixtures/journal_mode_test.sqlite3", pragmas: { "journal_mode" => false }) do |conn| + conn.execute("PRAGMA journal_mode") + end + end + assert_match(/unrecognized journal_mode false/, error.message) + end + end + + def test_overriding_default_synchronous_pragma + method_name = in_memory_db? ? :with_memory_connection : :with_file_connection + + send(method_name, pragmas: { synchronous: :full }) do |conn| + assert_equal [{ "synchronous" => 2 }], conn.execute("PRAGMA synchronous") + end + + send(method_name, pragmas: { synchronous: 2 }) do |conn| + assert_equal [{ "synchronous" => 2 }], conn.execute("PRAGMA synchronous") + end + + send(method_name, pragmas: { synchronous: "full" }) do |conn| + assert_equal [{ "synchronous" => 2 }], conn.execute("PRAGMA synchronous") + end + + error = assert_raises(ActiveRecord::StatementInvalid) do + send(method_name, pragmas: { synchronous: false }) do |conn| + conn.execute("PRAGMA synchronous") + end + end + assert_match(/unrecognized synchronous false/, error.message) + end + + def test_overriding_default_journal_size_limit_pragma + method_name = in_memory_db? ? :with_memory_connection : :with_file_connection + + send(method_name, pragmas: { journal_size_limit: 100 }) do |conn| + assert_equal [{ "journal_size_limit" => 100 }], conn.execute("PRAGMA journal_size_limit") + end + + send(method_name, pragmas: { journal_size_limit: "200" }) do |conn| + assert_equal [{ "journal_size_limit" => 200 }], conn.execute("PRAGMA journal_size_limit") + end + + error = assert_raises(ActiveRecord::StatementInvalid) do + send(method_name, pragmas: { journal_size_limit: false }) do |conn| + conn.execute("PRAGMA journal_size_limit") + end + end + assert_match(/undefined method `to_i'/, error.message) + + error = assert_raises(ActiveRecord::StatementInvalid) do + send(method_name, pragmas: { journal_size_limit: :false }) do |conn| + conn.execute("PRAGMA journal_size_limit") + end + end + assert_match(/undefined method `to_i'/, error.message) + end + + def test_overriding_default_mmap_size_pragma + # in-memory databases never have an mmap_size + if in_memory_db? + with_memory_connection(pragmas: { mmap_size: 100 }) do |conn| + assert_equal [], conn.execute("PRAGMA mmap_size") + end + + with_memory_connection(pragmas: { mmap_size: "200" }) do |conn| + assert_equal [], conn.execute("PRAGMA mmap_size") + end + + error = assert_raises(ActiveRecord::StatementInvalid) do + with_memory_connection(pragmas: { mmap_size: false }) do |conn| + conn.execute("PRAGMA mmap_size") + end + end + assert_match(/undefined method `to_i'/, error.message) + + error = assert_raises(ActiveRecord::StatementInvalid) do + with_memory_connection(pragmas: { mmap_size: :false }) do |conn| + conn.execute("PRAGMA mmap_size") + end + end + assert_match(/undefined method `to_i'/, error.message) + else + with_file_connection(pragmas: { mmap_size: 100 }) do |conn| + assert_equal [{ "mmap_size" => 100 }], conn.execute("PRAGMA mmap_size") + end + + with_file_connection(pragmas: { mmap_size: "200" }) do |conn| + assert_equal [{ "mmap_size" => 200 }], conn.execute("PRAGMA mmap_size") + end + + error = assert_raises(ActiveRecord::StatementInvalid) do + with_file_connection(pragmas: { mmap_size: false }) do |conn| + conn.execute("PRAGMA mmap_size") + end + end + assert_match(/undefined method `to_i'/, error.message) + + error = assert_raises(ActiveRecord::StatementInvalid) do + with_file_connection(pragmas: { mmap_size: :false }) do |conn| + conn.execute("PRAGMA mmap_size") + end + end + assert_match(/undefined method `to_i'/, error.message) + end + end + + def test_overriding_default_cache_size_pragma + method_name = in_memory_db? ? :with_memory_connection : :with_file_connection + + send(method_name, pragmas: { cache_size: 100 }) do |conn| + assert_equal [{ "cache_size" => 100 }], conn.execute("PRAGMA cache_size") + end + + send(method_name, pragmas: { cache_size: "200" }) do |conn| + assert_equal [{ "cache_size" => 200 }], conn.execute("PRAGMA cache_size") + end + + error = assert_raises(ActiveRecord::StatementInvalid) do + send(method_name, pragmas: { cache_size: false }) do |conn| + conn.execute("PRAGMA cache_size") + end + end + assert_match(/undefined method `to_i'/, error.message) + + error = assert_raises(ActiveRecord::StatementInvalid) do + send(method_name, pragmas: { cache_size: :false }) do |conn| + conn.execute("PRAGMA cache_size") + end + end + assert_match(/undefined method `to_i'/, error.message) + end + + def test_setting_new_pragma + if in_memory_db? + with_memory_connection(pragmas: { temp_store: :memory }) do |conn| + assert_equal [{ "foreign_keys" => 1 }], conn.execute("PRAGMA foreign_keys") + assert_equal [{ "journal_mode" => "memory" }], conn.execute("PRAGMA journal_mode") + assert_equal [{ "synchronous" => 1 }], conn.execute("PRAGMA synchronous") + assert_equal [{ "journal_size_limit" => 67108864 }], conn.execute("PRAGMA journal_size_limit") + assert_equal [], conn.execute("PRAGMA mmap_size") + assert_equal [{ "cache_size" => 2000 }], conn.execute("PRAGMA cache_size") + assert_equal [{ "temp_store" => 2 }], conn.execute("PRAGMA temp_store") + end + else + with_file_connection(pragmas: { temp_store: :memory }) do |conn| + assert_equal [{ "foreign_keys" => 1 }], conn.execute("PRAGMA foreign_keys") + assert_equal [{ "journal_mode" => "wal" }], conn.execute("PRAGMA journal_mode") + assert_equal [{ "synchronous" => 1 }], conn.execute("PRAGMA synchronous") + assert_equal [{ "journal_size_limit" => 67108864 }], conn.execute("PRAGMA journal_size_limit") + assert_equal [{ "mmap_size" => 134217728 }], conn.execute("PRAGMA mmap_size") + assert_equal [{ "cache_size" => 2000 }], conn.execute("PRAGMA cache_size") + assert_equal [{ "temp_store" => 2 }], conn.execute("PRAGMA temp_store") + end + end + end + + def test_setting_invalid_pragma + if in_memory_db? + warning = capture(:stderr) do + with_memory_connection(pragmas: { invalid: true }) do |conn| + conn.execute("PRAGMA foreign_keys") + end + end + assert_match(/Unknown SQLite pragma: invalid/, warning) + else + warning = capture(:stderr) do + with_file_connection(pragmas: { invalid: true }) do |conn| + conn.execute("PRAGMA foreign_keys") + end + end + assert_match(/Unknown SQLite pragma: invalid/, warning) + end + end + def test_exec_no_binds with_example_table "id int, data string" do result = @conn.exec_query("SELECT id, data FROM ex") @@ -868,7 +1107,7 @@ def with_strict_strings_by_default def with_file_connection(options = {}) options = options.dup - db_config = ActiveRecord::Base.configurations.configurations.find { |config| !config.database.include?(":memory:") } + db_config = ActiveRecord::Base.configurations.configs_for(env_name: "arunit2", name: "primary") options[:database] ||= db_config.database conn = SQLite3Adapter.new(options) @@ -876,6 +1115,16 @@ def with_file_connection(options = {}) ensure conn.disconnect! if conn end + + def with_memory_connection(options = {}) + options = options.dup + options[:database] = ":memory:" + conn = SQLite3Adapter.new(options) + + yield(conn) + ensure + conn.disconnect! if conn + end end end end