Merge pull request #770 from Infomaniak/entropyMaximizer

feat: truncate email too long for Realm

Mail/Views/Thread/MessageView+Preprocessing.swift

@@ -23,17 +23,16 @@ import MailCore
 
 /// MessageView code related to pre-processing
 extension MessageView {
-
     /// Maximum body size supported for preprocessing
     ///
     /// 1 Meg looks like a fine threshold
     private static let bodySizeThreshold = 1_000_000
-    
+
     /// Cooldown before processing each batch of inline images
     ///
     /// 4 seconds feels fine
     private static let batchCooldown: UInt64 = 4_000_000_000
-    
+
     // MARK: - public interface
 
     func prepareBodyIfNeeded() {
@@ -69,8 +68,9 @@ extension MessageView {
             return
         }
 
-        presentableBody.body = messageBody.detached()
-        let bodyValue = messageBody.value ?? ""
+        let detachedMessage = messageBody.detached()
+        presentableBody.body = detachedMessage
+        let bodyValue = detachedMessage.value ?? ""
 
         // Heuristic to give up on mail too large for "perfect" preprocessing.
         guard bodyValue.lengthOfBytes(using: String.Encoding.utf8) < Self.bodySizeThreshold else {

MailCore/Cache/MailboxManager.swift

@@ -72,7 +72,7 @@ public class MailboxManager: ObservableObject {
         let realmName = "\(mailbox.userId)-\(mailbox.mailboxId).realm"
         realmConfiguration = Realm.Configuration(
             fileURL: MailboxManager.constants.rootDocumentsURL.appendingPathComponent(realmName),
-            schemaVersion: 8,
+            schemaVersion: 9,
             deleteRealmIfMigrationNeeded: true,
             objectTypes: [
                 Folder.self,

MailCore/Models/Draft.swift

@@ -52,7 +52,7 @@ public struct DraftResponse: Codable {
     public var uid: String
 }
 
-public class Draft: Object, Decodable, Identifiable, Encodable {
+public class Draft: Object, Codable, Identifiable {
     @Persisted(primaryKey: true) public var localUUID = UUID().uuidString
     @Persisted public var remoteUUID = ""
     @Persisted public var date = Date()
@@ -63,7 +63,6 @@ public class Draft: Object, Decodable, Identifiable, Encodable {
     @Persisted public var references: String?
     @Persisted public var inReplyTo: String?
     @Persisted public var mimeType: String = UTType.html.preferredMIMEType!
-    @Persisted public var body = ""
     @Persisted public var to: List<Recipient>
     @Persisted public var cc: List<Recipient>
     @Persisted public var bcc: List<Recipient>
@@ -75,6 +74,27 @@ public class Draft: Object, Decodable, Identifiable, Encodable {
     @Persisted public var action: SaveDraftOption?
     @Persisted public var delay: Int?
 
+    /// Public facing "body", wrapping `bodyData`
+    public var body: String {
+        get {
+            guard let decompressedString = bodyData?.decompressedString() else {
+                return ""
+            }
+
+            return decompressedString
+        } set {
+            guard let data = newValue.compressed() else {
+                bodyData = nil
+                return
+            }
+
+            bodyData = data
+        }
+    }
+
+    /// Store compressed data to reduce realm size.
+    @Persisted var bodyData: Data?
+
     private enum CodingKeys: String, CodingKey {
         case remoteUUID = "uuid"
         case date
@@ -101,6 +121,13 @@ public class Draft: Object, Decodable, Identifiable, Encodable {
 
     public required init(from decoder: Decoder) throws {
         let values = try decoder.container(keyedBy: CodingKeys.self)
+
+        var buffer = try values.decode(String.self, forKey: .body)
+        buffer = String.truncatedForRealmIfNeeded(buffer)
+        if let compressedData = buffer.compressed() {
+            bodyData = compressedData
+        }
+
         remoteUUID = try values.decode(String.self, forKey: .remoteUUID)
         date = try values.decode(Date.self, forKey: .date)
         identityId = try values.decodeIfPresent(String.self, forKey: .identityId)
@@ -109,7 +136,6 @@ public class Draft: Object, Decodable, Identifiable, Encodable {
         references = try values.decodeIfPresent(String.self, forKey: .references)
         inReplyTo = try values.decodeIfPresent(String.self, forKey: .inReplyTo)
         mimeType = try values.decode(String.self, forKey: .mimeType)
-        body = try values.decode(String.self, forKey: .body)
         to = try values.decode(List<Recipient>.self, forKey: .to)
         cc = try values.decode(List<Recipient>.self, forKey: .cc)
         bcc = try values.decode(List<Recipient>.self, forKey: .bcc)

MailCore/Models/Message.swift

@@ -17,8 +17,38 @@
  */
 
 import Foundation
+import InfomaniakCore
 import MailResources
 import RealmSwift
+import Sentry
+
+// TODO move to core
+public extension String {
+    /// Max length of a string before we need to truncate it.
+    static let closeToMaxRealmSize = 14_000_000
+
+    /// Truncate a string for compatibility with Realm if needed
+    ///
+    /// The string will be terminated by " [truncated]" if it was
+    var truncatedForRealmIfNeeded: Self {
+        Self.truncatedForRealmIfNeeded(self)
+    }
+
+    /// Truncate a string for compatibility with Realm if needed
+    ///
+    /// The string will be terminated by " [truncated]" if it was
+    /// - Parameter input: an input string
+    /// - Returns: The output string truncated if needed
+    static func truncatedForRealmIfNeeded(_ input: String) -> String {
+        if input.utf8.count > Self.closeToMaxRealmSize {
+            let index = input.index(input.startIndex, offsetBy: Self.closeToMaxRealmSize)
+            let truncatedValue = String(input[...index]) + " [truncated]"
+            return truncatedValue
+        } else {
+            return input
+        }
+    }
+}
 
 public enum NewMessagesDirection: String {
     case previous
@@ -30,7 +60,7 @@ public struct PaginationInfo {
     let direction: NewMessagesDirection
 }
 
-public class MessageUidsResult: Decodable {
+public final class MessageUidsResult: Decodable {
     public let messageShortUids: [String]
     public let cursor: String
 
@@ -40,11 +70,11 @@ public class MessageUidsResult: Decodable {
     }
 }
 
-public class MessageByUidsResult: Decodable {
+public final class MessageByUidsResult: Decodable {
     public let messages: [Message]
 }
 
-public class MessageDeltaResult: Decodable {
+public final class MessageDeltaResult: Decodable {
     public let deletedShortUids: [String]
     public let addedShortUids: [String]
     public let updated: [MessageFlags]
@@ -111,7 +141,7 @@ public enum MessageDKIM: String, Codable, PersistableEnum {
     case notSigned = "not_signed"
 }
 
-public class Message: Object, Decodable, Identifiable {
+public final class Message: Object, Decodable, Identifiable {
     @Persisted(primaryKey: true) public var uid = ""
     @Persisted public var messageId: String?
     @Persisted public var subject: String?
@@ -287,7 +317,11 @@ public class Message: Object, Decodable, Identifiable {
         cc = try values.decode(List<Recipient>.self, forKey: .cc)
         bcc = try values.decode(List<Recipient>.self, forKey: .bcc)
         replyTo = try values.decode(List<Recipient>.self, forKey: .replyTo)
-        body = try values.decodeIfPresent(Body.self, forKey: .body)
+
+        /// Preprocessing body with a ProxyBody
+        let jsonBody = try values.decodeIfPresent(ProxyBody.self, forKey: .body)
+        body = jsonBody?.realmObject()
+
         if let attachments = try? values.decode(List<Attachment>.self, forKey: .attachments) {
             self.attachments = attachments
         } else {
@@ -411,10 +445,51 @@ public struct BodyResult: Codable {
     let body: Body
 }
 
-public class Body: EmbeddedObject, Codable {
-    @Persisted public var value: String?
+/// Proxy class to preprocess JSON of a Body object
+/// Preprocessing body to remain within Realm limitations
+final class ProxyBody: Codable {
+    public var value: String?
+    public var type: String?
+    public var subBody: String?
+
+    /// Generate a new persisted realm object on the fly
+    public func realmObject() -> Body {
+
+        // truncate message if needed
+        let truncatedValue = value?.truncatedForRealmIfNeeded
+
+        let body = Body()
+        body.value = truncatedValue
+        body.type = type
+        body.subBody = subBody
+        return body
+    }
+}
+
+public final class Body: EmbeddedObject, Codable {
+    /// Public facing "value", wrapping `valueData`
+    public var value: String? {
+        get {
+            guard let decompressedString = valueData?.decompressedString() else {
+                return nil
+            }
+
+            return decompressedString
+        } set {
+            guard let data = newValue?.compressed() else {
+                valueData = nil
+                return
+            }
+
+            valueData = data
+        }
+    }
+
     @Persisted public var type: String?
     @Persisted public var subBody: String?
+
+    /// Store compressed data to reduce realm size.
+    @Persisted var valueData: Data?
 }
 
 public struct MessageActionResult: Codable {

Project.swift

@@ -29,7 +29,7 @@ let project = Project(name: "Mail",
                       packages: [
                           .package(url: "https://github.com/Infomaniak/ios-login", .upToNextMajor(from: "4.0.0")),
                           .package(url: "https://github.com/Infomaniak/ios-dependency-injection", .upToNextMajor(from: "1.1.6")),
-                          .package(url: "https://github.com/Infomaniak/ios-core", .revision("cf001ba55dbf5f152353d66b55cd46350bd4b895")),
+                          .package(url: "https://github.com/Infomaniak/ios-core", .revision("e5fe8e03aa06375f20c8e40f4fae3a6af6e4d75e")),
                           .package(url: "https://github.com/Infomaniak/ios-core-ui", .upToNextMajor(from: "2.3.0")),
                           .package(url: "https://github.com/Infomaniak/ios-notifications", .upToNextMajor(from: "2.1.0")),
                           .package(url: "https://github.com/Infomaniak/ios-create-account", .upToNextMajor(from: "1.1.0")),