From 91ff20e5d254432a8ef17d5c6432bb1da8e31201 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Tue, 20 Sep 2022 14:10:52 -0700 Subject: [PATCH 1/5] CHANGELOG: mention that users should upgrade CLI --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 98382a5175..0c37be3155 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,7 +34,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * [[#2057]]: Make begin,commit,rollback cancel-safe in sqlite [[@madadam]] * [[#2058]]: fix typo in documentation [[@lovasoa]] * [[#2067]]: fix(docs): close code block in query_builder.rs [[@abonander]] -* [[#2069]]: Fix `prepare` race condition in workspaces [[@cycraig]] +* [[#2069]]: Fix `prepare` race condition in workspaces [[@cycraig]]\ + * NOTE: this changes the directory structure under `target/` that `cargo sqlx prepare` depends on. + If you use offline mode in your workflow, please rerun `cargo install sqlx-cli` to upgrade. * [[#2072]]: SqliteConnectOptions typo [[@fasterthanlime]] * [[#2074]]: fix: mssql uses unsigned for tinyint instead of signed [[@he4d]] * [[#2081]]: close unnamed portal after each executed extended query [[@DXist]] From c4e719133867b614cd18a2a95efe2a2ffbbe301c Mon Sep 17 00:00:00 2001 From: Miles Liu Date: Sat, 24 Sep 2022 15:47:55 +0800 Subject: [PATCH 2/5] [SQLite] Add option to execute `PRAGMA optimize;` on close of a connection --- sqlx-sqlite/src/connection/mod.rs | 13 +++++++ sqlx-sqlite/src/options/mod.rs | 60 +++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+) diff --git a/sqlx-sqlite/src/connection/mod.rs b/sqlx-sqlite/src/connection/mod.rs index 1fa53f38d7..2ea1b66ed9 100644 --- a/sqlx-sqlite/src/connection/mod.rs +++ b/sqlx-sqlite/src/connection/mod.rs @@ -11,8 +11,11 @@ use std::ptr::NonNull; use crate::connection::establish::EstablishParams; use crate::connection::worker::ConnectionWorker; +use crate::options::OptimizeOnClose; use crate::statement::VirtualStatement; use crate::{Sqlite, SqliteConnectOptions}; +use sqlx_core::executor::Executor; +use std::fmt::Write; pub(crate) use sqlx_core::connection::*; @@ -39,6 +42,7 @@ mod worker; /// You can explicitly call [`.close()`][Self::close] to ensure the database is closed successfully /// or get an error otherwise. pub struct SqliteConnection { + optimize_on_close: OptimizeOnClose, pub(crate) worker: ConnectionWorker, pub(crate) row_channel_size: usize, } @@ -70,6 +74,7 @@ impl SqliteConnection { let params = EstablishParams::from_options(options)?; let worker = ConnectionWorker::establish(params).await?; Ok(Self { + optimize_on_close: options.optimize_on_close.clone(), worker, row_channel_size: options.row_channel_size, }) @@ -102,6 +107,14 @@ impl Connection for SqliteConnection { fn close(mut self) -> BoxFuture<'static, Result<(), Error>> { Box::pin(async move { + if let OptimizeOnClose::Enabled { analysis_limit } = self.optimize_on_close { + let mut pragma_string = String::new(); + if let Some(limit) = analysis_limit { + write!(pragma_string, "PRAGMA analysis_limit = {}; ", limit).ok(); + } + pragma_string.push_str("PRAGMA optimize;"); + self.execute(&*pragma_string).await?; + } let shutdown = self.worker.shutdown(); // Drop the statement worker, which should // cover all references to the connection handle outside of the worker thread diff --git a/sqlx-sqlite/src/options/mod.rs b/sqlx-sqlite/src/options/mod.rs index 9b171e7770..99382874bb 100644 --- a/sqlx-sqlite/src/options/mod.rs +++ b/sqlx-sqlite/src/options/mod.rs @@ -79,6 +79,14 @@ pub struct SqliteConnectOptions { pub(crate) serialized: bool, pub(crate) thread_name: Arc String + Send + Sync + 'static>>, + + pub(crate) optimize_on_close: OptimizeOnClose, +} + +#[derive(Clone, Debug)] +pub enum OptimizeOnClose { + Enabled { analysis_limit: Option }, + Disabled, } impl Default for SqliteConnectOptions { @@ -167,6 +175,9 @@ impl SqliteConnectOptions { pragmas.insert("auto_vacuum".into(), None); + // Soft limit on the number of rows that `ANALYZE` touches per index. + pragmas.insert("analysis_limit".into(), None); + Self { filename: Cow::Borrowed(Path::new(":memory:")), in_memory: false, @@ -185,6 +196,7 @@ impl SqliteConnectOptions { thread_name: Arc::new(DebugFn(|id| format!("sqlx-sqlite-worker-{}", id))), command_channel_size: 50, row_channel_size: 50, + optimize_on_close: OptimizeOnClose::Disabled, } } @@ -458,4 +470,52 @@ impl SqliteConnectOptions { .insert(extension_name.into(), Some(entry_point.into())); self } + + /// Execute `PRAGMA optimize;` on the SQLite connection before closing. + /// + /// The SQLite manual recommends using this for long-lived databases. + /// + /// This will collect and store statistics about the layout of data in your tables to help the query planner make better decisions. + /// Over the connection's lifetime, the query planner will make notes about which tables could use up-to-date statistics so this + /// command doesn't have to scan the whole database every time. Thus, the best time to execute this is on connection close. + /// + /// `analysis_limit` sets a soft limit on the maximum number of rows to scan per index. + /// It is equivalent to setting [`Self::analysis_limit`] but only takes effect for the `PRAGMA optimize;` call + /// and does not affect the behavior of any `ANALYZE` statements made during the connection's lifetime. + /// + /// If not `None`, the `analysis_limit` here overrides the global `analysis_limit` setting, + /// but only for the `PRAGMA optimize;` call. + /// + /// Not enabled by default. + /// + /// See [the SQLite manual](https://www.sqlite.org/lang_analyze.html#automatically_running_analyze) for details. + pub fn optimize_on_close( + mut self, + enabled: bool, + analysis_limit: impl Into>, + ) -> Self { + self.optimize_on_close = if enabled { + OptimizeOnClose::Enabled { + analysis_limit: (analysis_limit.into()), + } + } else { + OptimizeOnClose::Disabled + }; + self + } + + /// Set a soft limit on the number of rows that `ANALYZE` touches per index. + /// + /// This also affects `PRAGMA optimize` which is set by [Self::optimize_on_close]. + /// + /// The value recommended by SQLite is `400`. There is no default. + /// + /// See [the SQLite manual](https://www.sqlite.org/lang_analyze.html#approx) for details. + pub fn analysis_limit(mut self, limit: impl Into>) -> Self { + if let Some(limit) = limit.into() { + return self.pragma("analysis_limit", limit.to_string()); + } + self.pragmas.insert("analysis_limit".into(), None); + self + } } From e552d8cfc05c519871051e35d67d3a436ebfcf24 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Tue, 14 Feb 2023 12:41:29 -0800 Subject: [PATCH 3/5] Update sqlx-sqlite/src/options/mod.rs --- sqlx-sqlite/src/options/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlx-sqlite/src/options/mod.rs b/sqlx-sqlite/src/options/mod.rs index d17be6ede0..4f0590a870 100644 --- a/sqlx-sqlite/src/options/mod.rs +++ b/sqlx-sqlite/src/options/mod.rs @@ -90,7 +90,7 @@ pub struct SqliteConnectOptions { pub enum OptimizeOnClose { Enabled { analysis_limit: Option }, Disabled, - +} impl Default for SqliteConnectOptions { fn default() -> Self { From dba396a2b2c0524626c750c08f7645600b06590c Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Tue, 14 Feb 2023 12:41:58 -0800 Subject: [PATCH 4/5] Update sqlx-sqlite/src/options/mod.rs --- sqlx-sqlite/src/options/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/sqlx-sqlite/src/options/mod.rs b/sqlx-sqlite/src/options/mod.rs index 4f0590a870..851c6d5e4a 100644 --- a/sqlx-sqlite/src/options/mod.rs +++ b/sqlx-sqlite/src/options/mod.rs @@ -521,6 +521,7 @@ impl SqliteConnectOptions { return self.pragma("analysis_limit", limit.to_string()); } self.pragmas.insert("analysis_limit".into(), None); + self } /// Register a regexp function that allows using regular expressions in queries. From 89f40e16e1b4d4be945b7a6788dd4e97ec8050b3 Mon Sep 17 00:00:00 2001 From: Miles Liu Date: Wed, 15 Feb 2023 09:05:09 +0800 Subject: [PATCH 5/5] Update sqlx-sqlite/src/options/mod.rs --- sqlx-sqlite/src/options/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sqlx-sqlite/src/options/mod.rs b/sqlx-sqlite/src/options/mod.rs index 851c6d5e4a..21013547ba 100644 --- a/sqlx-sqlite/src/options/mod.rs +++ b/sqlx-sqlite/src/options/mod.rs @@ -81,7 +81,7 @@ pub struct SqliteConnectOptions { pub(crate) thread_name: Arc String + Send + Sync + 'static>>, pub(crate) optimize_on_close: OptimizeOnClose, - + #[cfg(feature = "regexp")] pub(crate) register_regexp_function: bool, } @@ -523,7 +523,7 @@ impl SqliteConnectOptions { self.pragmas.insert("analysis_limit".into(), None); self } - + /// Register a regexp function that allows using regular expressions in queries. /// /// ```