Allow overriding SQLite defaults from database.yml (#50460)

* Allow overriding SQLite defaults from `database.yml`

Any PRAGMA configuration set under the `pragmas` key in the configuration file take precedence over Rails' defaults, and additional PRAGMAs can be set as well.

```yaml
database: storage/development.sqlite3
timeout: 5000
pragmas:
  synchronous: full
  temp_store: memory
```

* Style

* Allow overriding SQLite defaults from `database.yml`

Any PRAGMA configuration set under the `pragmas` key in the configuration file take precedence over Rails' defaults, and additional PRAGMAs can be set as well.

```yaml
database: storage/development.sqlite3
timeout: 5000
pragmas:
  synchronous: full
  temp_store: memory
```

---------

Co-authored-by: David Heinemeier Hansson <david@hey.com>

activerecord/CHANGELOG.md

@@ -1,3 +1,17 @@
+*   Allow overriding SQLite defaults from `database.yml`
+
+    Any PRAGMA configuration set under the `pragmas` key in the configuration file take precedence over Rails' defaults, and additional PRAGMAs can be set as well.
+
+    ```yaml
+    database: storage/development.sqlite3
+    timeout: 5000
+    pragmas:
+      journal_mode: off
+      temp_store: memory
+    ```
+
+    *Stephen Margheim*
+
 *   Remove warning message when running SQLite in production, but leave it unconfigured
 
     There are valid use cases for running SQLite in production, however it must be done

activerecord/lib/active_record/connection_adapters/sqlite3_adapter.rb

@@ -78,6 +78,15 @@ def dbconsole(config, options = {})
         json:         { name: "json" },
       }
 
+      DEFAULT_PRAGMAS = {
+        "foreign_keys"        => true,
+        "journal_mode"        => :wal,
+        "synchronous"         => :normal,
+        "mmap_size"           => 134217728, # 128 megabytes
+        "journal_size_limit"  => 67108864, # 64 megabytes
+        "cache_size"          => 2000
+      }
+
       class StatementPool < ConnectionAdapters::StatementPool # :nodoc:
         alias reset clear
 
@@ -759,29 +768,14 @@ def configure_connection
 
           super
 
-          # Enforce foreign key constraints
-          # https://www.sqlite.org/pragma.html#pragma_foreign_keys
-          # https://www.sqlite.org/foreignkeys.html
-          raw_execute("PRAGMA foreign_keys = ON", "SCHEMA")
-          unless @memory_database
-            # Journal mode WAL allows for greater concurrency (many readers + one writer)
-            # https://www.sqlite.org/pragma.html#pragma_journal_mode
-            raw_execute("PRAGMA journal_mode = WAL", "SCHEMA")
-            # Set more relaxed level of database durability
-            # 2 = "FULL" (sync on every write), 1 = "NORMAL" (sync every 1000 written pages) and 0 = "NONE"
-            # https://www.sqlite.org/pragma.html#pragma_synchronous
-            raw_execute("PRAGMA synchronous = NORMAL", "SCHEMA")
-            # Set the global memory map so all processes can share some data
-            # https://www.sqlite.org/pragma.html#pragma_mmap_size
-            # https://www.sqlite.org/mmap.html
-            raw_execute("PRAGMA mmap_size = #{128.megabytes}", "SCHEMA")
+          pragmas = @config.fetch(:pragmas, {}).stringify_keys
+          DEFAULT_PRAGMAS.merge(pragmas).each do |pragma, value|
+            if ::SQLite3::Pragmas.method_defined?("#{pragma}=")
+              @raw_connection.public_send("#{pragma}=", value)
+            else
+              warn "Unknown SQLite pragma: #{pragma}"
+            end
           end
-          # Impose a limit on the WAL file to prevent unlimited growth
-          # https://www.sqlite.org/pragma.html#pragma_journal_size_limit
-          raw_execute("PRAGMA journal_size_limit = #{64.megabytes}", "SCHEMA")
-          # Set the local connection cache to 2000 pages
-          # https://www.sqlite.org/pragma.html#pragma_cache_size
-          raw_execute("PRAGMA cache_size = 2000", "SCHEMA")
         end
     end
     ActiveSupport.run_load_hooks(:active_record_sqlite3adapter, SQLite3Adapter)

activerecord/test/cases/adapters/sqlite3/sqlite3_adapter_test.rb

@@ -159,7 +159,7 @@ def test_default_pragmas
         if in_memory_db?
           assert_equal [{ "foreign_keys" => 1 }], @conn.execute("PRAGMA foreign_keys")
           assert_equal [{ "journal_mode" => "memory" }], @conn.execute("PRAGMA journal_mode")
-          assert_equal [{ "synchronous" => 2 }], @conn.execute("PRAGMA synchronous")
+          assert_equal [{ "synchronous" => 1 }], @conn.execute("PRAGMA synchronous")
           assert_equal [{ "journal_size_limit" => 67108864 }], @conn.execute("PRAGMA journal_size_limit")
           assert_equal [], @conn.execute("PRAGMA mmap_size")
           assert_equal [{ "cache_size" => 2000 }], @conn.execute("PRAGMA cache_size")
@@ -175,6 +175,245 @@ def test_default_pragmas
         end
       end
 
+      def test_overriding_default_foreign_keys_pragma
+        method_name = in_memory_db? ? :with_memory_connection : :with_file_connection
+
+        send(method_name, pragmas: { foreign_keys: false }) do |conn|
+          assert_equal [{ "foreign_keys" => 0 }], conn.execute("PRAGMA foreign_keys")
+        end
+
+        send(method_name, pragmas: { foreign_keys: 0 }) do |conn|
+          assert_equal [{ "foreign_keys" => 0 }], conn.execute("PRAGMA foreign_keys")
+        end
+
+        send(method_name, pragmas: { foreign_keys: "false" }) do |conn|
+          assert_equal [{ "foreign_keys" => 0 }], conn.execute("PRAGMA foreign_keys")
+        end
+
+        error = assert_raises(ActiveRecord::StatementInvalid) do
+          send(method_name, pragmas: { foreign_keys: :false }) do |conn|
+            conn.execute("PRAGMA foreign_keys")
+          end
+        end
+        assert_match(/unrecognized pragma parameter :false/, error.message)
+      end
+
+      def test_overriding_default_journal_mode_pragma
+        # in-memory databases are always only ever in `memory` journal_mode
+        if in_memory_db?
+          with_memory_connection(pragmas: { "journal_mode" => "delete" }) do |conn|
+            assert_equal [{ "journal_mode" => "memory" }], conn.execute("PRAGMA journal_mode")
+          end
+
+          with_memory_connection(pragmas: { "journal_mode" => :delete }) do |conn|
+            assert_equal [{ "journal_mode" => "memory" }], conn.execute("PRAGMA journal_mode")
+          end
+
+          error = assert_raises(ActiveRecord::StatementInvalid) do
+            with_memory_connection(pragmas: { "journal_mode" => 0 }) do |conn|
+              conn.execute("PRAGMA journal_mode")
+            end
+          end
+          assert_match(/nrecognized journal_mode 0/, error.message)
+
+          error = assert_raises(ActiveRecord::StatementInvalid) do
+            with_memory_connection(pragmas: { "journal_mode" => false }) do |conn|
+              conn.execute("PRAGMA journal_mode")
+            end
+          end
+          assert_match(/nrecognized journal_mode false/, error.message)
+        else
+          # must use a new, separate database file that hasn't been opened in WAL mode before
+          with_file_connection(database: "fixtures/journal_mode_test.sqlite3", pragmas: { "journal_mode" => "delete" }) do |conn|
+            assert_equal [{ "journal_mode" => "delete" }], conn.execute("PRAGMA journal_mode")
+          end
+
+          with_file_connection(database: "fixtures/journal_mode_test.sqlite3", pragmas: { "journal_mode" => :delete }) do |conn|
+            assert_equal [{ "journal_mode" => "delete" }], conn.execute("PRAGMA journal_mode")
+          end
+
+          error = assert_raises(ActiveRecord::StatementInvalid) do
+            with_file_connection(database: "fixtures/journal_mode_test.sqlite3", pragmas: { "journal_mode" => 0 }) do |conn|
+              conn.execute("PRAGMA journal_mode")
+            end
+          end
+          assert_match(/unrecognized journal_mode 0/, error.message)
+
+          error = assert_raises(ActiveRecord::StatementInvalid) do
+            with_file_connection(database: "fixtures/journal_mode_test.sqlite3", pragmas: { "journal_mode" => false }) do |conn|
+              conn.execute("PRAGMA journal_mode")
+            end
+          end
+          assert_match(/unrecognized journal_mode false/, error.message)
+        end
+      end
+
+      def test_overriding_default_synchronous_pragma
+        method_name = in_memory_db? ? :with_memory_connection : :with_file_connection
+
+        send(method_name, pragmas: { synchronous: :full }) do |conn|
+          assert_equal [{ "synchronous" => 2 }], conn.execute("PRAGMA synchronous")
+        end
+
+        send(method_name, pragmas: { synchronous: 2 }) do |conn|
+          assert_equal [{ "synchronous" => 2 }], conn.execute("PRAGMA synchronous")
+        end
+
+        send(method_name, pragmas: { synchronous: "full" }) do |conn|
+          assert_equal [{ "synchronous" => 2 }], conn.execute("PRAGMA synchronous")
+        end
+
+        error = assert_raises(ActiveRecord::StatementInvalid) do
+          send(method_name, pragmas: { synchronous: false }) do |conn|
+            conn.execute("PRAGMA synchronous")
+          end
+        end
+        assert_match(/unrecognized synchronous false/, error.message)
+      end
+
+      def test_overriding_default_journal_size_limit_pragma
+        method_name = in_memory_db? ? :with_memory_connection : :with_file_connection
+
+        send(method_name, pragmas: { journal_size_limit: 100 }) do |conn|
+          assert_equal [{ "journal_size_limit" => 100 }], conn.execute("PRAGMA journal_size_limit")
+        end
+
+        send(method_name, pragmas: { journal_size_limit: "200" }) do |conn|
+          assert_equal [{ "journal_size_limit" => 200 }], conn.execute("PRAGMA journal_size_limit")
+        end
+
+        error = assert_raises(ActiveRecord::StatementInvalid) do
+          send(method_name, pragmas: { journal_size_limit: false }) do |conn|
+            conn.execute("PRAGMA journal_size_limit")
+          end
+        end
+        assert_match(/undefined method `to_i'/, error.message)
+
+        error = assert_raises(ActiveRecord::StatementInvalid) do
+          send(method_name, pragmas: { journal_size_limit: :false }) do |conn|
+            conn.execute("PRAGMA journal_size_limit")
+          end
+        end
+        assert_match(/undefined method `to_i'/, error.message)
+      end
+
+      def test_overriding_default_mmap_size_pragma
+        # in-memory databases never have an mmap_size
+        if in_memory_db?
+          with_memory_connection(pragmas: { mmap_size: 100 }) do |conn|
+            assert_equal [], conn.execute("PRAGMA mmap_size")
+          end
+
+          with_memory_connection(pragmas: { mmap_size: "200" }) do |conn|
+            assert_equal [], conn.execute("PRAGMA mmap_size")
+          end
+
+          error = assert_raises(ActiveRecord::StatementInvalid) do
+            with_memory_connection(pragmas: { mmap_size: false }) do |conn|
+              conn.execute("PRAGMA mmap_size")
+            end
+          end
+          assert_match(/undefined method `to_i'/, error.message)
+
+          error = assert_raises(ActiveRecord::StatementInvalid) do
+            with_memory_connection(pragmas: { mmap_size: :false }) do |conn|
+              conn.execute("PRAGMA mmap_size")
+            end
+          end
+          assert_match(/undefined method `to_i'/, error.message)
+        else
+          with_file_connection(pragmas: { mmap_size: 100 }) do |conn|
+            assert_equal [{ "mmap_size" => 100 }], conn.execute("PRAGMA mmap_size")
+          end
+
+          with_file_connection(pragmas: { mmap_size: "200" }) do |conn|
+            assert_equal [{ "mmap_size" => 200 }], conn.execute("PRAGMA mmap_size")
+          end
+
+          error = assert_raises(ActiveRecord::StatementInvalid) do
+            with_file_connection(pragmas: { mmap_size: false }) do |conn|
+              conn.execute("PRAGMA mmap_size")
+            end
+          end
+          assert_match(/undefined method `to_i'/, error.message)
+
+          error = assert_raises(ActiveRecord::StatementInvalid) do
+            with_file_connection(pragmas: { mmap_size: :false }) do |conn|
+              conn.execute("PRAGMA mmap_size")
+            end
+          end
+          assert_match(/undefined method `to_i'/, error.message)
+        end
+      end
+
+      def test_overriding_default_cache_size_pragma
+        method_name = in_memory_db? ? :with_memory_connection : :with_file_connection
+
+        send(method_name, pragmas: { cache_size: 100 }) do |conn|
+          assert_equal [{ "cache_size" => 100 }], conn.execute("PRAGMA cache_size")
+        end
+
+        send(method_name, pragmas: { cache_size: "200" }) do |conn|
+          assert_equal [{ "cache_size" => 200 }], conn.execute("PRAGMA cache_size")
+        end
+
+        error = assert_raises(ActiveRecord::StatementInvalid) do
+          send(method_name, pragmas: { cache_size: false }) do |conn|
+            conn.execute("PRAGMA cache_size")
+          end
+        end
+        assert_match(/undefined method `to_i'/, error.message)
+
+        error = assert_raises(ActiveRecord::StatementInvalid) do
+          send(method_name, pragmas: { cache_size: :false }) do |conn|
+            conn.execute("PRAGMA cache_size")
+          end
+        end
+        assert_match(/undefined method `to_i'/, error.message)
+      end
+
+      def test_setting_new_pragma
+        if in_memory_db?
+          with_memory_connection(pragmas: { temp_store: :memory }) do |conn|
+            assert_equal [{ "foreign_keys" => 1 }], conn.execute("PRAGMA foreign_keys")
+            assert_equal [{ "journal_mode" => "memory" }], conn.execute("PRAGMA journal_mode")
+            assert_equal [{ "synchronous" => 1 }], conn.execute("PRAGMA synchronous")
+            assert_equal [{ "journal_size_limit" => 67108864 }], conn.execute("PRAGMA journal_size_limit")
+            assert_equal [], conn.execute("PRAGMA mmap_size")
+            assert_equal [{ "cache_size" => 2000 }], conn.execute("PRAGMA cache_size")
+            assert_equal [{ "temp_store" => 2 }], conn.execute("PRAGMA temp_store")
+          end
+        else
+          with_file_connection(pragmas: { temp_store: :memory }) do |conn|
+            assert_equal [{ "foreign_keys" => 1 }], conn.execute("PRAGMA foreign_keys")
+            assert_equal [{ "journal_mode" => "wal" }], conn.execute("PRAGMA journal_mode")
+            assert_equal [{ "synchronous" => 1 }], conn.execute("PRAGMA synchronous")
+            assert_equal [{ "journal_size_limit" => 67108864 }], conn.execute("PRAGMA journal_size_limit")
+            assert_equal [{ "mmap_size" => 134217728 }], conn.execute("PRAGMA mmap_size")
+            assert_equal [{ "cache_size" => 2000 }], conn.execute("PRAGMA cache_size")
+            assert_equal [{ "temp_store" => 2 }], conn.execute("PRAGMA temp_store")
+          end
+        end
+      end
+
+      def test_setting_invalid_pragma
+        if in_memory_db?
+          warning = capture(:stderr) do
+            with_memory_connection(pragmas: { invalid: true }) do |conn|
+              conn.execute("PRAGMA foreign_keys")
+            end
+          end
+          assert_match(/Unknown SQLite pragma: invalid/, warning)
+        else
+          warning = capture(:stderr) do
+            with_file_connection(pragmas: { invalid: true }) do |conn|
+              conn.execute("PRAGMA foreign_keys")
+            end
+          end
+          assert_match(/Unknown SQLite pragma: invalid/, warning)
+        end
+      end
+
       def test_exec_no_binds
         with_example_table "id int, data string" do
           result = @conn.exec_query("SELECT id, data FROM ex")
@@ -868,14 +1107,24 @@ def with_strict_strings_by_default
 
         def with_file_connection(options = {})
           options = options.dup
-          db_config = ActiveRecord::Base.configurations.configurations.find { |config| !config.database.include?(":memory:") }
+          db_config = ActiveRecord::Base.configurations.configs_for(env_name: "arunit2", name: "primary")
           options[:database] ||= db_config.database
           conn = SQLite3Adapter.new(options)
 
           yield(conn)
         ensure
           conn.disconnect! if conn
         end
+
+        def with_memory_connection(options = {})
+          options = options.dup
+          options[:database] = ":memory:"
+          conn = SQLite3Adapter.new(options)
+
+          yield(conn)
+        ensure
+          conn.disconnect! if conn
+        end
     end
   end
 end