Merge branch 'development'

CHANGELOG.md

@@ -7,6 +7,7 @@ GRDB adheres to [Semantic Versioning](https://semver.org/), with one exception:
 
 #### 5.x Releases
 
+- `5.12.x` Releases - [5.12.0](#5120)
 - `5.11.x` Releases - [5.11.0](#5110)
 - `5.10.x` Releases - [5.10.0](#5100)
 - `5.9.x` Releases - [5.9.0](#590)
@@ -78,6 +79,15 @@ GRDB adheres to [Semantic Versioning](https://semver.org/), with one exception:
 
 ---
 
+## 5.12.0
+
+Released September 25, 2021 • [diff](https://github.com/groue/GRDB.swift/compare/v5.11.0...v5.12.0)
+
+- **Breaking Change**: Minimum iOS version is now iOS 11.0, and 32-bits devices are no longer supported. This fixes issues introduced by Xcode 13 ([#1033](https://github.com/groue/GRDB.swift/issues/1033), [#1059](https://github.com/groue/GRDB.swift/issues/1059)).
+- **New**: `FTS3Pattern(matchingAllPrefixesIn:)` and `FTS5Pattern(matchingAllPrefixesIn:)` let you build full-text search patterns suitable for prefix queries.
+- **New**: `FTS3.tokenize(_:)` and `FTS5Tokenizer.tokenize(_:)` let you tokenize input strings.
+- **Documentation Update**: Full-text search documentation was updated for the new search pattern initializers ([FTS3/4](Documentation/FullTextSearch.md#fts3pattern), [FTS5](Documentation/FullTextSearch.md#fts5pattern)), and tokenization methods ([FTS3/4](Documentation/FullTextSearch.md#fts3-and-fts4-tokenization), [FTS5](Documentation/FullTextSearch.md#fts5-tokenization)).
+
 ## 5.11.0
 
 Released September 3, 2021 • [diff](https://github.com/groue/GRDB.swift/compare/v5.10.0...v5.11.0)

Documentation/FullTextSearch.md

@@ -31,6 +31,7 @@ let books = try Book.fetchAll(db,
 - **[Enabling FTS5 Support](#enabling-fts5-support)**
 - **Create Full-Text Virtual Tables**: [FTS3/4](#create-fts3-and-fts4-virtual-tables), [FTS5](#create-fts5-virtual-tables)
 - **Choosing a Tokenizer**: [FTS3/4](#fts3-and-fts4-tokenizers), [FTS5](#fts5-tokenizers)
+- **Tokenization**: [FTS3/4](#fts3-and-fts4-tokenization), [FTS5](#fts5-tokenization)
 - **Search Patterns**: [FTS3/4](#fts3pattern), [FTS5](#fts5pattern)
 - **Sorting by Relevance**: [FTS5](#fts5-sorting-by-relevance)
 - **External Content Full-Text Tables**: [FTS4/5](#external-content-full-text-tables)
@@ -229,6 +230,21 @@ See below some examples of matches:
 See [SQLite tokenizers](https://www.sqlite.org/fts3.html#tokenizer) for more information.
 
 
+## FTS3 and FTS4 Tokenization
+
+You can tokenize strings when needed:
+
+```swift
+// Default tokenization using the `simple` tokenizer:
+FTS3.tokenize("SQLite database")  // ["sqlite", "database"]
+FTS3.tokenize("Gustave Doré")     // ["gustave", "doré"])
+
+// Tokenization with an explicit tokenizer:
+FTS3.tokenize("SQLite database", withTokenizer: .porter)   // ["sqlite", "databas"]
+FTS3.tokenize("Gustave Doré", withTokenizer: .unicode61()) // ["gustave", "dore"])
+```
+
+
 ## FTS3Pattern
 
 **Full-text search in FTS3 and FTS4 tables is performed with search patterns:**
@@ -248,6 +264,7 @@ struct FTS3Pattern {
     init(rawPattern: String) throws
     init?(matchingAnyTokenIn string: String)
     init?(matchingAllTokensIn string: String)
+    init?(matchingAllPrefixesIn string: String)
     init?(matchingPhrase string: String)
 }
 ```
@@ -257,18 +274,25 @@ The first initializer validates your raw patterns against the query grammar, and
 ```swift
 // OK: FTS3Pattern
 let pattern = try FTS3Pattern(rawPattern: "sqlite AND database")
+
 // DatabaseError: malformed MATCH expression: [AND]
 let pattern = try FTS3Pattern(rawPattern: "AND")
 ```
 
-The three other initializers don't throw. They build a valid pattern from any string, **including strings provided by users of your application**. They let you find documents that match all given words, any given word, or a full phrase, depending on the needs of your application:
+The other initializers don't throw. They build a valid pattern from any string, **including strings provided by users of your application**. They let you find documents that match any given word, all given words or prefixes, or a full phrase, depending on the needs of your application:
 
 ```swift
 let query = "SQLite database"
+
 // Matches documents that contain "SQLite" or "database"
 let pattern = FTS3Pattern(matchingAnyTokenIn: query)
-// Matches documents that contain both "SQLite" and "database"
+
+// Matches documents that contain "SQLite" and "database"
 let pattern = FTS3Pattern(matchingAllTokensIn: query)
+
+// Matches documents that contain words that start with "SQLite" and words that start with "database"
+let pattern = FTS3Pattern(matchingAllPrefixesIn: query)
+
 // Matches documents that contain "SQLite database"
 let pattern = FTS3Pattern(matchingPhrase: query)
 ```
@@ -280,7 +304,7 @@ let pattern = FTS3Pattern(matchingAnyTokenIn: "")  // nil
 let pattern = FTS3Pattern(matchingAnyTokenIn: "*") // nil
 ```
 
-FTS3Pattern are regular [values](../README.md#values). You can use them as query [arguments](http://groue.github.io/GRDB.swift/docs/5.11/Structs/StatementArguments.html):
+FTS3Pattern are regular [values](../README.md#values). You can use them as query [arguments](http://groue.github.io/GRDB.swift/docs/5.12/Structs/StatementArguments.html):
 
 ```swift
 let documents = try Document.fetchAll(db,
@@ -444,7 +468,7 @@ See below some examples of matches:
 
     ```swift
     try db.create(virtualTable: "book", using: FTS5()) { t in
-        t.tokenizer = .ascii
+        t.tokenizer = .ascii()
     }
     ```
 
@@ -458,8 +482,8 @@ See below some examples of matches:
 
     ```swift
     try db.create(virtualTable: "book", using: FTS5()) { t in
-        t.tokenizer = .porter()       // porter wrapping unicode61 (the default)
-        t.tokenizer = .porter(.ascii) // porter wrapping ascii
+        t.tokenizer = .porter()         // porter wrapping unicode61 (the default)
+        t.tokenizer = .porter(.ascii()) // porter wrapping ascii
         t.tokenizer = .porter(.unicode61(diacritics: .keep)) // porter wrapping unicode61 without diacritics stripping
     }
     ```
@@ -471,6 +495,29 @@ See below some examples of matches:
 See [SQLite tokenizers](https://www.sqlite.org/fts5.html#tokenizers) for more information, and [custom FTS5 tokenizers](FTS5Tokenizers.md) in order to add your own tokenizers.
 
 
+## FTS5 Tokenization
+
+You can tokenize strings when needed:
+
+```swift
+let ascii = try db.makeTokenizer(.ascii())
+
+// Tokenize an FTS5 query
+for (token, flags) in try ascii.tokenize(query: "SQLite database") {
+    print(token) // Prints "sqlite" then "database"
+}
+
+// Tokenize an FTS5 document
+for (token, flags) in try ascii.tokenize(document: "SQLite database") {
+    print(token) // Prints "sqlite" then "database"
+}
+```
+
+Some tokenizers may produce a different output when you tokenize a query or a document (see `FTS5_TOKENIZE_QUERY` and `FTS5_TOKENIZE_DOCUMENT` in https://www.sqlite.org/fts5.html#custom_tokenizers). You should generally use `tokenize(query:)` when you intend to tokenize a string in order to compose a [raw search pattern](#fts5pattern).
+
+See the `FTS5_TOKEN_*` flags in https://www.sqlite.org/fts5.html#custom_tokenizers for more information about token flags. In particular, tokenizers that support synonyms may output multiple tokens for a single word, along with the `.colocated` flag.
+
+
 ## FTS5Pattern
 
 **Full-text search in FTS5 tables is performed with search patterns:**
@@ -493,7 +540,9 @@ extension Database {
 struct FTS5Pattern {
     init?(matchingAnyTokenIn string: String)
     init?(matchingAllTokensIn string: String)
+    init?(matchingAllPrefixesIn string: String)
     init?(matchingPhrase string: String)
+    init?(matchingPrefixPhrase string: String)
 }
 ```
 
@@ -502,8 +551,10 @@ The `Database.makeFTS5Pattern(rawPattern:forTable:)` method validates your raw p
 ```swift
 // OK: FTS5Pattern
 try db.makeFTS5Pattern(rawPattern: "sqlite", forTable: "book")
+
 // DatabaseError: syntax error near \"AND\"
 try db.makeFTS5Pattern(rawPattern: "AND", forTable: "book")
+
 // DatabaseError: no such column: missing
 try db.makeFTS5Pattern(rawPattern: "missing: sqlite", forTable: "book")
 ```
@@ -512,12 +563,19 @@ The FTS5Pattern initializers don't throw. They build a valid pattern from any st
 
 ```swift
 let query = "SQLite database"
+
 // Matches documents that contain "SQLite" or "database"
 let pattern = FTS5Pattern(matchingAnyTokenIn: query)
-// Matches documents that contain both "SQLite" and "database"
+
+// Matches documents that contain "SQLite" and "database"
 let pattern = FTS5Pattern(matchingAllTokensIn: query)
+
+// Matches documents that contain words that start with "SQLite" and words that start with "database"
+let pattern = FTS5Pattern(matchingAllPrefixesIn: query)
+
 // Matches documents that contain "SQLite database"
 let pattern = FTS5Pattern(matchingPhrase: query)
+
 // Matches documents that start with "SQLite database"
 let pattern = FTS5Pattern(matchingPrefixPhrase: query)
 ```
@@ -529,7 +587,7 @@ let pattern = FTS5Pattern(matchingAnyTokenIn: "")  // nil
 let pattern = FTS5Pattern(matchingAnyTokenIn: "*") // nil
 ```
 
-FTS5Pattern are regular [values](../README.md#values). You can use them as query [arguments](http://groue.github.io/GRDB.swift/docs/5.11/Structs/StatementArguments.html):
+FTS5Pattern are regular [values](../README.md#values). You can use them as query [arguments](http://groue.github.io/GRDB.swift/docs/5.12/Structs/StatementArguments.html):
 
 ```swift
 let documents = try Document.fetchAll(db,

Documentation/GRDB5MigrationGuide.md

@@ -26,7 +26,7 @@ GRDB requirements have been bumped:
 
 - **Swift 5.2+** (was Swift 4.2+)
 - **Xcode 11.4+** (was Xcode 10.0+)
-- **iOS 10.0+** (was iOS 9.0+)
+- **iOS 11.0+** (was iOS 9.0+)
 - **macOS 10.10+** (was macOS 10.9+)
 - tvOS 9.0+ (unchanged)
 - watchOS 2.0+ (unchanged)

Documentation/Migrations.md

@@ -68,7 +68,7 @@ try dbQueue.read { db in
 }
 ```
 
-See the [DatabaseMigrator reference](http://groue.github.io/GRDB.swift/docs/5.11/Structs/DatabaseMigrator.html) for more migrator methods.
+See the [DatabaseMigrator reference](http://groue.github.io/GRDB.swift/docs/5.12/Structs/DatabaseMigrator.html) for more migrator methods.
 
 
 ## The `eraseDatabaseOnSchemaChange` Option

GRDB.swift.podspec

@@ -1,6 +1,6 @@
 Pod::Spec.new do |s|
   s.name     = 'GRDB.swift'
-  s.version  = '5.11.0'
+  s.version  = '5.12.0'
 
   s.license  = { :type => 'MIT', :file => 'LICENSE' }
   s.summary  = 'A toolkit for SQLite databases, with a focus on application development.'
@@ -10,7 +10,7 @@ Pod::Spec.new do |s|
   s.module_name = 'GRDB'
 
   s.swift_versions = ['5.2']
-  s.ios.deployment_target = '10.0'
+  s.ios.deployment_target = '11.0'
   s.osx.deployment_target = '10.10'
   s.watchos.deployment_target = '2.0'
   s.tvos.deployment_target = '9.0'

GRDB/Core/Support/Foundation/DatabaseDateComponents.swift

@@ -84,8 +84,7 @@ public struct DatabaseDateComponents: DatabaseValueConvertible, StatementColumnC
         guard let components = optionalComponents else {
             return nil
         }
-        self.dateComponents = components.dateComponents
-        self.format = components.format
+        self.init(components.dateComponents, format: components.format)
     }
 
     // MARK: - DatabaseValueConvertible adoption

GRDB/FTS/FTS3.swift

@@ -37,7 +37,41 @@ public struct FTS3: VirtualTableModule {
     ///     try db.create(virtualTable: "document", using: FTS3()) { t in
     ///         t.column("content")
     ///     }
-    public init() {
+    public init() { }
+
+    /// Returns an array of tokens found in the string argument.
+    ///
+    /// For example:
+    ///
+    ///     FTS3.tokenize("SQLite database")  // ["sqlite", "database"]
+    ///     FTS3.tokenize("Gustave Doré")     // ["gustave", "doré"])
+    ///
+    /// Results can be altered with an explicit tokenizer - default is `.simple`.
+    /// See <https://www.sqlite.org/fts3.html#tokenizer>.
+    ///
+    ///     FTS3.tokenize("SQLite database", withTokenizer: .porter)   // ["sqlite", "databas"]
+    ///     FTS3.tokenize("Gustave Doré", withTokenizer: .unicode61()) // ["gustave", "dore"])
+    ///
+    /// Tokenization is performed by the `fts3tokenize` virtual table described
+    /// at <https://www.sqlite.org/fts3.html#querying_tokenizers>.
+    public static func tokenize(
+        _ string: String,
+        withTokenizer tokenizer: FTS3TokenizerDescriptor = .simple)
+    -> [String]
+    {
+        DatabaseQueue().inDatabase { db in
+            var tokenizerChunks: [String] = []
+            tokenizerChunks.append(tokenizer.name)
+            for option in tokenizer.arguments {
+                tokenizerChunks.append("\"\(option)\"")
+            }
+            let tokenizerSQL = tokenizerChunks.joined(separator: ", ")
+            // Assume fts3tokenize virtual table in an in-memory database always succeeds
+            try! db.execute(sql: "CREATE VIRTUAL TABLE tokens USING fts3tokenize(\(tokenizerSQL))")
+            return try! String.fetchAll(db, sql: """
+                SELECT token FROM tokens WHERE input = ? ORDER BY position
+                """, arguments: [string])
+        }
     }
 
     // MARK: - VirtualTableModule Adoption

GRDB/FTS/FTS3Pattern.swift

@@ -41,7 +41,7 @@ public struct FTS3Pattern {
     ///
     /// - parameter string: The string to turn into an FTS3 pattern
     public init?(matchingAnyTokenIn string: String) {
-        let tokens = FTS3TokenizerDescriptor.simple.tokenize(string)
+        let tokens = FTS3.tokenize(string, withTokenizer: .simple)
         guard !tokens.isEmpty else { return nil }
         try? self.init(rawPattern: tokens.joined(separator: " OR "))
     }
@@ -54,11 +54,24 @@ public struct FTS3Pattern {
     ///
     /// - parameter string: The string to turn into an FTS3 pattern
     public init?(matchingAllTokensIn string: String) {
-        let tokens = FTS3TokenizerDescriptor.simple.tokenize(string)
+        let tokens = FTS3.tokenize(string, withTokenizer: .simple)
         guard !tokens.isEmpty else { return nil }
         try? self.init(rawPattern: tokens.joined(separator: " "))
     }
 
+    /// Creates a pattern that matches all token prefixes found in the input
+    /// string; returns nil if no pattern could be built.
+    ///
+    ///     FTS3Pattern(matchingAllTokensIn: "")        // nil
+    ///     FTS3Pattern(matchingAllTokensIn: "foo bar") // foo* bar*
+    ///
+    /// - parameter string: The string to turn into an FTS3 pattern
+    public init?(matchingAllPrefixesIn string: String) {
+        let tokens = FTS3.tokenize(string, withTokenizer: .simple)
+        guard !tokens.isEmpty else { return nil }
+        try? self.init(rawPattern: tokens.map { "\($0)*" }.joined(separator: " "))
+    }
+
     /// Creates a pattern that matches a contiguous string; returns nil if no
     /// pattern could be built.
     ///
@@ -67,7 +80,7 @@ public struct FTS3Pattern {
     ///
     /// - parameter string: The string to turn into an FTS3 pattern
     public init?(matchingPhrase string: String) {
-        let tokens = FTS3TokenizerDescriptor.simple.tokenize(string)
+        let tokens = FTS3.tokenize(string, withTokenizer: .simple)
         guard !tokens.isEmpty else { return nil }
         try? self.init(rawPattern: "\"" + tokens.joined(separator: " ") + "\"")
     }

GRDB/FTS/FTS3TokenizerDescriptor.swift

@@ -90,27 +90,4 @@ public struct FTS3TokenizerDescriptor {
         }
         return FTS3TokenizerDescriptor("unicode61", arguments: arguments)
     }
-
-    func tokenize(_ string: String) -> [String] {
-        _tokenize(string)
-    }
-
-    /// Returns an array of tokens found in the string argument.
-    ///
-    ///     FTS3TokenizerDescriptor.simple.tokenize("foo bar") // ["foo", "bar"]
-    private func _tokenize(_ string: String) -> [String] {
-        DatabaseQueue().inDatabase { db in
-            var tokenizerChunks: [String] = []
-            tokenizerChunks.append(name)
-            for option in arguments {
-                tokenizerChunks.append("\"\(option)\"")
-            }
-            let tokenizerSQL = tokenizerChunks.joined(separator: ", ")
-            // Assume fts3tokenize virtual table in an in-memory database always succeeds
-            try! db.execute(sql: "CREATE VIRTUAL TABLE tokens USING fts3tokenize(\(tokenizerSQL))")
-            return try! String.fetchAll(db, sql: """
-                SELECT token FROM tokens WHERE input = ? ORDER BY position
-                """, arguments: [string])
-        }
-    }
 }

GRDB/FTS/FTS5.swift

@@ -44,7 +44,31 @@ public struct FTS5: VirtualTableModule {
     ///     }
     ///
     /// See <https://www.sqlite.org/fts5.html>
-    public init() {
+    public init() { }
+
+    // Support for FTS5Pattern initializers. Don't make public. Users tokenize
+    // with `FTS5Tokenizer.tokenize()` methods, which support custom tokenizers,
+    // token flags, and query/document tokenzation.
+    /// Tokenizes the string argument as an FTS5 query.
+    ///
+    /// For example:
+    ///
+    ///     try FTS5.tokenize(query: "SQLite database")  // ["sqlite", "database"]
+    ///     try FTS5.tokenize(query: "Gustave Doré")     // ["gustave", "doré"])
+    ///
+    /// Synonym (colocated) tokens are not present in the returned array. See
+    /// `FTS5_TOKEN_COLOCATED` at <https://www.sqlite.org/fts5.html#custom_tokenizers>
+    /// for more information.
+    ///
+    /// - parameter string: The tokenized string.
+    /// - returns: An array of tokens.
+    /// - throws: An error if tokenization fails.
+    static func tokenize(query string: String) throws -> [String] {
+        try DatabaseQueue().inDatabase { db in
+            try db.makeTokenizer(.ascii()).tokenize(query: string).compactMap {
+                $0.flags.contains(.colocated) ? nil : $0.token
+            }
+        }
     }
 
     // MARK: - VirtualTableModule Adoption