feat: truncate email too long for Realm

Mail/Views/Thread/MessageView+Preprocessing.swift

@@ -23,17 +23,16 @@ import MailCore
 
 /// MessageView code related to pre-processing
 extension MessageView {
-
     /// Maximum body size supported for preprocessing
     ///
     /// 1 Meg looks like a fine threshold
     private static let bodySizeThreshold = 1_000_000
-    
+
     /// Cooldown before processing each batch of inline images
     ///
     /// 4 seconds feels fine
     private static let batchCooldown: UInt64 = 4_000_000_000
-    
+
     // MARK: - public interface
 
     func prepareBodyIfNeeded() {
@@ -69,8 +68,9 @@ extension MessageView {
             return
         }
 
-        presentableBody.body = messageBody.detached()
-        let bodyValue = messageBody.value ?? ""
+        let detachedMessage = messageBody.detached()
+        presentableBody.body = detachedMessage
+        let bodyValue = detachedMessage.value ?? ""
 
         // Heuristic to give up on mail too large for "perfect" preprocessing.
         guard bodyValue.lengthOfBytes(using: String.Encoding.utf8) < Self.bodySizeThreshold else {

MailCore/Cache/MailboxManager.swift

@@ -72,7 +72,7 @@ public class MailboxManager: ObservableObject {
         let realmName = "\(mailbox.userId)-\(mailbox.mailboxId).realm"
         realmConfiguration = Realm.Configuration(
             fileURL: MailboxManager.constants.rootDocumentsURL.appendingPathComponent(realmName),
-            schemaVersion: 8,
+            schemaVersion: 9,
             deleteRealmIfMigrationNeeded: true,
             objectTypes: [
                 Folder.self,

MailCore/Models/Message.swift

@@ -19,6 +19,53 @@
 import Foundation
 import MailResources
 import RealmSwift
+import Sentry
+
+// TODO: move to core
+/// LZFSE Wrapper
+public extension Data {
+    /// Compressed data using a zstd like algorithm: lzfse
+    func compressed() -> Self? {
+        guard let data = try? (self as NSData).compressed(using: .lzfse) as Data else {
+            return nil
+        }
+        return data
+    }
+
+    /// Decompressed data from a lzfse buffer
+    func decompressed() -> Self? {
+        guard let data = try? (self as NSData).decompressed(using: .lzfse) as Data else {
+            return nil
+        }
+        return data
+    }
+
+    // MARK: - String helpers
+
+    /// Decompressed string from a lzfse buffer
+    func decompressedString() -> String? {
+        guard let decompressedData = decompressed() else {
+            return nil
+        }
+        let string = String(decoding: decompressedData, as: UTF8.self)
+        return string
+    }
+}
+
+/// LZFSE Wrapper
+public extension String {
+    func compressed() -> Data? {
+        guard let data = data(using: .utf8),
+              let compressed = data.compressed() else {
+            return nil
+        }
+        return compressed
+    }
+
+    static func decompressed(from data: Data) -> Self? {
+        data.decompressedString()
+    }
+}
 
 public enum NewMessagesDirection: String {
     case previous
@@ -30,7 +77,7 @@ public struct PaginationInfo {
     let direction: NewMessagesDirection
 }
 
-public class MessageUidsResult: Decodable {
+public final class MessageUidsResult: Decodable {
     public let messageShortUids: [String]
     public let cursor: String
 
@@ -40,11 +87,11 @@ public class MessageUidsResult: Decodable {
     }
 }
 
-public class MessageByUidsResult: Decodable {
+public final class MessageByUidsResult: Decodable {
     public let messages: [Message]
 }
 
-public class MessageDeltaResult: Decodable {
+public final class MessageDeltaResult: Decodable {
     public let deletedShortUids: [String]
     public let addedShortUids: [String]
     public let updated: [MessageFlags]
@@ -111,7 +158,7 @@ public enum MessageDKIM: String, Codable, PersistableEnum {
     case notSigned = "not_signed"
 }
 
-public class Message: Object, Decodable, Identifiable {
+public final class Message: Object, Decodable, Identifiable {
     @Persisted(primaryKey: true) public var uid = ""
     @Persisted public var messageId: String?
     @Persisted public var subject: String?
@@ -286,7 +333,11 @@ public class Message: Object, Decodable, Identifiable {
         cc = try values.decode(List<Recipient>.self, forKey: .cc)
         bcc = try values.decode(List<Recipient>.self, forKey: .bcc)
         replyTo = try values.decode(List<Recipient>.self, forKey: .replyTo)
-        body = try values.decodeIfPresent(Body.self, forKey: .body)
+
+        /// Preprocessing body with a ProxyBody
+        let jsonBody = try values.decodeIfPresent(ProxyBody.self, forKey: .body)
+        body = jsonBody?.realmObject()
+
         if let attachments = try? values.decode(List<Attachment>.self, forKey: .attachments) {
             self.attachments = attachments
         } else {
@@ -410,10 +461,60 @@ public struct BodyResult: Codable {
     let body: Body
 }
 
-public class Body: EmbeddedObject, Codable {
-    @Persisted public var value: String?
+/// Proxy class to preprocess JSON of a Body object
+/// Preprocessing body to remain within Realm limitations
+final class ProxyBody: Codable {
+    public var value: String?
+    public var type: String?
+    public var subBody: String?
+
+    /// Max length of a message before we truncate it.
+    static let tenMegabytes = 10_000_000
+
+    /// Generate a new persisted realm object on the fly
+    public func realmObject() -> Body {
+        let truncatedValue: String?
+        // truncate message, if more text than 10MB (realm breaks at 15)
+        if let value = value,
+           value.count > Self.tenMegabytes {
+            let index = value.index(value.startIndex, offsetBy: Self.tenMegabytes)
+            truncatedValue = String(value[...index]) + " [truncated]"
+        } else {
+            truncatedValue = value
+        }
+
+        let body = Body()
+        body.value = truncatedValue
+        body.type = type
+        body.subBody = subBody
+        return body
+    }
+}
+
+public final class Body: EmbeddedObject, Codable {
+    /// Public facing "value", wrapping `valueData`
+    public var value: String? {
+        get {
+            guard let decompressedString = valueData?.decompressedString() else {
+                return nil
+            }
+
+            return decompressedString
+        } set {
+            guard let data = newValue?.compressed() else {
+                valueData = nil
+                return
+            }
+
+            valueData = data
+        }
+    }
+
     @Persisted public var type: String?
     @Persisted public var subBody: String?
+
+    // Store compressed data to reduce realm size.
+    @Persisted var valueData: Data?
 }
 
 public struct MessageActionResult: Codable {