[iOS] Include origin directory in backup if it is visited after certa…

…in period of time

https://bugs.webkit.org/show_bug.cgi?id=242605
rdar://94855540

Reviewed by Geoffrey Garen.

In iOS 16, we introduced origin directory and origin file in new website data layout. This leads to more directories
and files are backed up for WebKit apps, increasing backup size and putting pressure on backup server. To solve the
issue, we now only back up origin directories if user visits the origin multiple times.
Specifically, what this patch does are:
1. mark origin directory excluded from backup at its creation
2. mark origin directory included in backup when origin directory is visited after backup exclusion period (currently
the period is 24 hours)

* Source/WTF/wtf/FileSystem.cpp:
(WTF::FileSystemImpl::setExcludedFromBackup):
(WTF::FileSystemImpl::excludeFromBackup): Deleted.
* Source/WTF/wtf/FileSystem.h:
* Source/WTF/wtf/cocoa/FileSystemCocoa.mm:
(WTF::FileSystemImpl::setExcludedFromBackup):
(WTF::FileSystemImpl::excludeFromBackup): Deleted.
* Source/WebKit/NetworkProcess/NetworkProcess.h:
* Source/WebKit/NetworkProcess/NetworkProcess.messages.in:
* Source/WebKit/NetworkProcess/cocoa/NetworkProcessCocoa.mm:
(WebKit::NetworkProcess::setBackupExclusionPeriodForTesting):
* Source/WebKit/NetworkProcess/storage/NetworkStorageManager.cpp:
(WebKit::writeOriginToFile):
(WebKit::NetworkStorageManager::NetworkStorageManager):
(WebKit::NetworkStorageManager::includeOriginInBackupIfNecessary):
(WebKit::NetworkStorageManager::writeOriginToFileIfNecessary):
(WebKit::NetworkStorageManager::setBackupExclusionPeriodForTesting):
* Source/WebKit/NetworkProcess/storage/NetworkStorageManager.h:
* Source/WebKit/NetworkProcess/storage/OriginStorageManager.cpp:
(WebKit::OriginStorageManager::StorageBucket::resolvedLocalStoragePath):
* Source/WebKit/NetworkProcess/storage/OriginStorageManager.h:
(WebKit::OriginStorageManager::originFileCreationTimestamp const):
(WebKit::OriginStorageManager::setOriginFileCreationTimestamp):
(WebKit::OriginStorageManager::includedInBackup const):
(WebKit::OriginStorageManager::markIncludedInBackup):
(WebKit::OriginStorageManager::didWriteOriginToFile const): Deleted.
(WebKit::OriginStorageManager::markDidWriteOriginToFile): Deleted.
* Source/WebKit/UIProcess/API/Cocoa/WKWebsiteDataStore.mm:
(-[WKWebsiteDataStore _setBackupExclusionPeriodForTesting:completionHandler:]):
* Source/WebKit/UIProcess/API/Cocoa/WKWebsiteDataStorePrivate.h:
* Source/WebKit/UIProcess/Network/NetworkProcessProxy.h:
* Source/WebKit/UIProcess/Network/NetworkProcessProxyCocoa.mm:
(WebKit::NetworkProcessProxy::setBackupExclusionPeriodForTesting):
* Source/WebKit/UIProcess/WebsiteData/Cocoa/WebsiteDataStoreCocoa.mm:
(WebKit::WebsiteDataStore::setBackupExclusionPeriodForTesting):
* Source/WebKit/UIProcess/WebsiteData/WebsiteDataStore.h:
* Tools/TestWebKitAPI/Tests/WebKitCocoa/LocalStoragePersistence.mm:
(TEST):
* Tools/TestWebKitAPI/Tests/WebKitCocoa/WebsiteDataStoreCustomPaths.mm:
(TEST):

Canonical link: https://commits.webkit.org/252472@main

Source/WTF/wtf/FileSystem.cpp

@@ -424,7 +424,7 @@ String createTemporaryZipArchive(const String&)
     return { };
 }
 
-bool excludeFromBackup(const String&)
+bool setExcludedFromBackup(const String&, bool)
 {
     return false;
 }

Source/WTF/wtf/FileSystem.h

@@ -134,7 +134,7 @@ WTF_EXPORT_PRIVATE std::optional<FileType> fileType(const String&);
 WTF_EXPORT_PRIVATE std::optional<FileType> fileTypeFollowingSymlinks(const String&);
 
 WTF_EXPORT_PRIVATE void setMetadataURL(const String& path, const String& urlString, const String& referrer = { });
-WTF_EXPORT_PRIVATE bool excludeFromBackup(const String&); // Returns true if successful.
+WTF_EXPORT_PRIVATE bool setExcludedFromBackup(const String&, bool); // Returns true if successful.
 
 WTF_EXPORT_PRIVATE Vector<String> listDirectory(const String& path); // Returns file names, not full paths.
 

Source/WTF/wtf/cocoa/FileSystemCocoa.mm

@@ -225,10 +225,10 @@ bool makeSafeToUseMemoryMapForPath(const String& path)
 }
 #endif
 
-bool excludeFromBackup(const String& path)
+bool setExcludedFromBackup(const String& path, bool excluded)
 {
     NSError *error;
-    if (![[NSURL fileURLWithPath:(NSString *)path isDirectory:YES] setResourceValue:@YES forKey:NSURLIsExcludedFromBackupKey error:&error]) {
+    if (![[NSURL fileURLWithPath:(NSString *)path isDirectory:YES] setResourceValue:[NSNumber numberWithBool:excluded] forKey:NSURLIsExcludedFromBackupKey error:&error]) {
         LOG_ERROR("Cannot exclude path '%s' from backup with error '%@'", path.utf8().data(), error.localizedDescription);
         return false;
     }

Source/WebKit/NetworkProcess/NetworkProcess.h

@@ -290,6 +290,9 @@ class NetworkProcess : public AuxiliaryProcess, private DownloadManager::Client,
     void syncLocalStorage(CompletionHandler<void()>&&);
 
     void resetQuota(PAL::SessionID, CompletionHandler<void()>&&);
+#if PLATFORM(IOS_FAMILY)
+    void setBackupExclusionPeriodForTesting(PAL::SessionID, Seconds, CompletionHandler<void()>&&);
+#endif
     void clearStorage(PAL::SessionID, CompletionHandler<void()>&&);
     void didIncreaseQuota(PAL::SessionID, WebCore::ClientOrigin&&, QuotaIncreaseRequestIdentifier, std::optional<uint64_t> newQuota);
     void renameOriginInWebsiteData(PAL::SessionID, const URL&, const URL&, OptionSet<WebsiteDataType>, CompletionHandler<void()>&&);

Source/WebKit/NetworkProcess/NetworkProcess.messages.in

@@ -176,6 +176,9 @@ messages -> NetworkProcess LegacyReceiver {
     ResetServiceWorkerFetchTimeoutForTesting() -> () Synchronous
 
     ResetQuota(PAL::SessionID sessionID) -> ()
+#if PLATFORM(IOS_FAMILY)
+    SetBackupExclusionPeriodForTesting(PAL::SessionID sessionID, Seconds period) -> ()
+#endif
     ClearStorage(PAL::SessionID sessionID) -> ()
     DidIncreaseQuota(PAL::SessionID sessionID, struct WebCore::ClientOrigin origin, WebKit::QuotaIncreaseRequestIdentifier identifier, std::optional<uint64_t> newQuota)
 

Source/WebKit/NetworkProcess/cocoa/NetworkProcessCocoa.mm

@@ -32,6 +32,7 @@
 #import "NetworkProcessCreationParameters.h"
 #import "NetworkResourceLoader.h"
 #import "NetworkSessionCocoa.h"
+#import "NetworkStorageManager.h"
 #import "SandboxExtension.h"
 #import "WebCookieManager.h"
 #import <WebCore/NetworkStorageSession.h>
@@ -256,4 +257,15 @@ void saveCookies(NSHTTPCookieStorage *cookieStorage, CompletionHandler<void()>&&
     return m_uiProcessBundleIdentifier;
 }
 
+#if PLATFORM(IOS_FAMILY)
+
+void NetworkProcess::setBackupExclusionPeriodForTesting(PAL::SessionID sessionID, Seconds period, CompletionHandler<void()>&& completionHandler)
+{
+    auto callbackAggregator = CallbackAggregator::create(WTFMove(completionHandler));
+    if (auto* session = networkSession(sessionID))
+        session->storageManager().setBackupExclusionPeriodForTesting(period, [callbackAggregator] { });
+}
+
+#endif
+
 } // namespace WebKit

Source/WebKit/NetworkProcess/storage/NetworkStorageManager.cpp

@@ -54,6 +54,10 @@
 
 namespace WebKit {
 
+#if PLATFORM(IOS_FAMILY)
+static const Seconds defaultBackupExclusionPeriod { 24_h };
+#endif
+
 static std::optional<WebCore::ClientOrigin> readOriginFromFile(const String& filePath)
 {
     ASSERT(!RunLoop::isMain());
@@ -79,10 +83,10 @@ static std::optional<WebCore::ClientOrigin> readOriginFromFile(const String& fil
     return origin;
 }
 
-static void writeOriginToFile(const String& filePath, const WebCore::ClientOrigin& origin)
+static bool writeOriginToFile(const String& filePath, const WebCore::ClientOrigin& origin)
 {
     if (filePath.isEmpty() || FileSystem::fileExists(filePath))
-        return;
+        return false;
 
     FileSystem::makeAllDirectories(FileSystem::parentPath(filePath));
     auto originFileHandle = FileSystem::openFile(filePath, FileSystem::FileOpenMode::ReadWrite);
@@ -92,12 +96,13 @@ static void writeOriginToFile(const String& filePath, const WebCore::ClientOrigi
 
     if (!FileSystem::isHandleValid(originFileHandle)) {
         LOG_ERROR("writeOriginToFile: Failed to open origin file '%s'", filePath.utf8().data());
-        return;
+        return false;
     }
 
     WTF::Persistence::Encoder encoder;
     encoder << origin;
     FileSystem::writeToFile(originFileHandle, encoder.buffer(), encoder.bufferSize());
+    return true;
 }
 
 static String encode(const String& string, FileSystem::Salt salt)
@@ -183,6 +188,9 @@ NetworkStorageManager::NetworkStorageManager(PAL::SessionID sessionID, IPC::Conn
     , m_defaultOriginQuota(defaultOriginQuota)
     , m_defaultThirdPartyOriginQuota(defaultThirdPartyOriginQuota)
     , m_parentConnection(connection)
+#if PLATFORM(IOS_FAMILY)
+    , m_backupExclusionPeriod(defaultBackupExclusionPeriod)
+#endif
 {
     ASSERT(RunLoop::isMain());
 
@@ -200,6 +208,13 @@ NetworkStorageManager::NetworkStorageManager(PAL::SessionID sessionID, IPC::Conn
             auto saltPath = FileSystem::pathByAppendingComponent(m_path, "salt"_s);
             m_salt = valueOrDefault(FileSystem::readOrMakeSalt(saltPath));
         }
+#if PLATFORM(IOS_FAMILY)
+        // Exclude LocalStorage directory to reduce backup traffic. See https://webkit.org/b/168388.
+        if (m_shouldUseCustomPaths && !m_customLocalStoragePath.isEmpty()) {
+            FileSystem::makeAllDirectories(m_customLocalStoragePath);
+            FileSystem::setExcludedFromBackup(m_customLocalStoragePath, true);
+        }
+#endif
     });
 }
 
@@ -273,23 +288,58 @@ void NetworkStorageManager::stopReceivingMessageFromConnection(IPC::Connection&
     });
 }
 
+#if PLATFORM(IOS_FAMILY)
+
+void NetworkStorageManager::includeOriginInBackupIfNecessary(OriginStorageManager& manager)
+{
+    if (manager.includedInBackup())
+        return;
+
+    auto originFileCreationTimestamp = manager.originFileCreationTimestamp();
+    if (!originFileCreationTimestamp)
+        return;
+
+    if (WallTime::now() - originFileCreationTimestamp.value() < m_backupExclusionPeriod)
+        return;
+
+    FileSystem::setExcludedFromBackup(manager.path(), false);
+    manager.markIncludedInBackup();
+}
+
+#endif
+
 void NetworkStorageManager::writeOriginToFileIfNecessary(const WebCore::ClientOrigin& origin, StorageAreaBase* storageArea)
 {
     auto* manager = m_localOriginStorageManagers.get(origin);
-    if (!manager || manager->didWriteOriginToFile())
+    if (!manager)
         return;
 
-    auto originDirectory = originDirectoryPath(m_path, origin, m_salt);
+    if (manager->originFileCreationTimestamp()) {
+#if PLATFORM(IOS_FAMILY)
+        includeOriginInBackupIfNecessary(*manager);
+#endif
+        return;
+    }
+
+    auto originDirectory = manager->path();
     if (originDirectory.isEmpty())
         return;
 
-    if (storageArea) {
-        if (isEmptyOriginDirectory(originDirectory))
-            return;
-    }
+    if (storageArea && isEmptyOriginDirectory(originDirectory))
+        return;
 
-    writeOriginToFile(originFilePath(originDirectory), origin);
-    manager->markDidWriteOriginToFile();
+    auto originFile = originFilePath(originDirectory);
+    bool didWrite = writeOriginToFile(originFile, origin);
+    auto timestamp = FileSystem::fileCreationTime(originFile);
+    manager->setOriginFileCreationTimestamp(timestamp);
+#if PLATFORM(IOS_FAMILY)
+    if (didWrite)
+        FileSystem::setExcludedFromBackup(originDirectory, true);
+    else
+        includeOriginInBackupIfNecessary(*manager);
+#else
+    UNUSED_PARAM(didWrite);
+#endif
 }
 
 OriginStorageManager& NetworkStorageManager::localOriginStorageManager(const WebCore::ClientOrigin& origin, ShouldWriteOriginFile shouldWriteOriginFile)
@@ -809,6 +859,20 @@ void NetworkStorageManager::resetQuotaUpdatedBasedOnUsageForTesting(WebCore::Cli
     });
 }
 
+#if PLATFORM(IOS_FAMILY)
+
+void NetworkStorageManager::setBackupExclusionPeriodForTesting(Seconds period, CompletionHandler<void()>&& completionHandler)
+{
+    ASSERT(RunLoop::isMain());
+
+    m_queue->dispatch([this, protectedThis = Ref { *this }, period, completionHandler = WTFMove(completionHandler)]() mutable {
+        m_backupExclusionPeriod = period;
+        RunLoop::main().dispatch(WTFMove(completionHandler));
+    });
+}
+
+#endif
+
 void NetworkStorageManager::connectToStorageArea(IPC::Connection& connection, WebCore::StorageType type, StorageAreaMapIdentifier sourceIdentifier, StorageNamespaceIdentifier namespaceIdentifier, const WebCore::ClientOrigin& origin, CompletionHandler<void(StorageAreaIdentifier, HashMap<String, String>, uint64_t)>&& completionHandler)
 {
     ASSERT(!RunLoop::isMain());

Source/WebKit/NetworkProcess/storage/NetworkStorageManager.h

@@ -98,6 +98,9 @@ class NetworkStorageManager final : public IPC::Connection::WorkQueueMessageRece
     void requestSpace(const WebCore::ClientOrigin&, uint64_t size, CompletionHandler<void(bool)>&&);
     void resetQuotaForTesting(CompletionHandler<void()>&&);
     void resetQuotaUpdatedBasedOnUsageForTesting(WebCore::ClientOrigin&&);
+#if PLATFORM(IOS_FAMILY)
+    void setBackupExclusionPeriodForTesting(Seconds, CompletionHandler<void()>&&);
+#endif
 
 private:
     NetworkStorageManager(PAL::SessionID, IPC::Connection::UniqueID, const String& path, const String& customLocalStoragePath, const String& customIDBStoragePath, const String& customCacheStoragePath, uint64_t defaultOriginQuota, uint64_t defaultThirdPartyOriginQuota, bool shouldUseCustomPaths);
@@ -111,6 +114,9 @@ class NetworkStorageManager final : public IPC::Connection::WorkQueueMessageRece
     HashSet<WebCore::ClientOrigin> getAllOrigins();
     Vector<WebsiteData::Entry> fetchDataFromDisk(OptionSet<WebsiteDataType>);
     HashSet<WebCore::ClientOrigin> deleteDataOnDisk(OptionSet<WebsiteDataType>, WallTime, const Function<bool(const WebCore::ClientOrigin&)>&);
+#if PLATFORM(IOS_FAMILY)
+    void includeOriginInBackupIfNecessary(OriginStorageManager&);
+#endif
 
     // IPC::MessageReceiver (implemented by generated code)
     void didReceiveMessage(IPC::Connection&, IPC::Decoder&);
@@ -190,6 +196,9 @@ class NetworkStorageManager final : public IPC::Connection::WorkQueueMessageRece
     bool m_shouldUseCustomPaths;
     IPC::Connection::UniqueID m_parentConnection;
     HashMap<IPC::Connection::UniqueID, HashSet<String>> m_temporaryBlobPathsByConnection;
+#if PLATFORM(IOS_FAMILY)
+    Seconds m_backupExclusionPeriod;
+#endif
 };
 
 } // namespace WebKit

Source/WebKit/NetworkProcess/storage/OriginStorageManager.cpp

@@ -386,7 +386,6 @@ String OriginStorageManager::StorageBucket::resolvedLocalStoragePath()
 
     if (m_shouldUseCustomPaths) {
         ASSERT(m_customLocalStoragePath.isEmpty() == m_rootPath.isEmpty());
-        FileSystem::makeAllDirectories(FileSystem::parentPath(m_customLocalStoragePath));
         m_resolvedLocalStoragePath = m_customLocalStoragePath;
     } else if (!m_rootPath.isEmpty()) {
         auto localStorageDirectory = typeStoragePath(StorageType::LocalStorage);
@@ -403,12 +402,6 @@ String OriginStorageManager::StorageBucket::resolvedLocalStoragePath()
     } else
         m_resolvedLocalStoragePath = emptyString();
 
-#if PLATFORM(IOS_FAMILY)
-    // Exclude LocalStorage directory to reduce backup traffic. See https://webkit.org/b/168388.
-    if (!m_resolvedLocalStoragePath.isEmpty())
-        FileSystem::excludeFromBackup(FileSystem::parentPath(m_customLocalStoragePath));
-#endif
-
     return m_resolvedLocalStoragePath;
 }
 

Source/WebKit/NetworkProcess/storage/OriginStorageManager.h

@@ -66,9 +66,13 @@ class OriginStorageManager {
     OptionSet<WebsiteDataType> fetchDataTypesInList(OptionSet<WebsiteDataType>);
     void deleteData(OptionSet<WebsiteDataType>, WallTime);
     void moveData(OptionSet<WebsiteDataType>, const String& localStoragePath, const String& idbStoragePath);
-    bool didWriteOriginToFile() const { return m_didWriteOriginToFile; }
-    void markDidWriteOriginToFile() { m_didWriteOriginToFile = true; }
     void deleteEmptyDirectory();
+    std::optional<WallTime> originFileCreationTimestamp() const { return m_originFileCreationTimestamp; }
+    void setOriginFileCreationTimestamp(std::optional<WallTime> timestamp) { m_originFileCreationTimestamp = timestamp; }
+#if PLATFORM(IOS_FAMILY)
+    bool includedInBackup() const { return m_includedInBackup; }
+    void markIncludedInBackup() { m_includedInBackup = true; }
+#endif
 
 private:
     enum class StorageBucketMode : bool;
@@ -85,7 +89,10 @@ class OriginStorageManager {
     RefPtr<QuotaManager> m_quotaManager;
     bool m_persisted { false };
     bool m_shouldUseCustomPaths;
-    bool m_didWriteOriginToFile { false };
+    std::optional<WallTime> m_originFileCreationTimestamp;
+#if PLATFORM(IOS_FAMILY)
+    bool m_includedInBackup { false };
+#endif
 };
 
 } // namespace WebKit

Source/WebKit/UIProcess/API/Cocoa/WKWebsiteDataStore.mm

@@ -916,4 +916,17 @@ -(void)_originDirectoryForTesting:(NSURL *)origin topOrigin:(NSURL *)topOrigin t
     });
 }
 
+-(void)_setBackupExclusionPeriodForTesting:(double)seconds completionHandler:(void(^)(void))completionHandler
+{
+#if PLATFORM(IOS_FAMILY)
+    _websiteDataStore->setBackupExclusionPeriodForTesting(Seconds(seconds), [completionHandler = makeBlockPtr(completionHandler)] {
+        completionHandler();
+    });
+#else
+    UNUSED_PARAM(seconds);
+    completionHandler();
+#endif
+}
+
+
 @end

Source/WebKit/UIProcess/API/Cocoa/WKWebsiteDataStorePrivate.h

@@ -117,6 +117,7 @@ typedef NS_OPTIONS(NSUInteger, _WKWebsiteDataStoreFetchOptions) {
 -(void)_getOriginsWithPushAndNotificationPermissions:(void(^)(NSSet<WKSecurityOrigin *> *))completionHandler WK_API_AVAILABLE(macos(13.0), ios(16.0));
 -(void)_scopeURL:(NSURL *)scopeURL hasPushSubscriptionForTesting:(void(^)(BOOL))completionHandler WK_API_AVAILABLE(macos(13.0), ios(16.0));
 -(void)_originDirectoryForTesting:(NSURL *)origin topOrigin:(NSURL *)topOrigin type:(NSString *)dataType completionHandler:(void(^)(NSString *))completionHandler WK_API_AVAILABLE(macos(13.0), ios(16.0));
+-(void)_setBackupExclusionPeriodForTesting:(double)seconds completionHandler:(void(^)(void))completionHandler WK_API_AVAILABLE(macos(13.0), ios(16.0));
 @end
 
 NS_ASSUME_NONNULL_END

Source/WebKit/UIProcess/Network/NetworkProcessProxy.h

@@ -249,6 +249,9 @@ class NetworkProcessProxy final : public AuxiliaryProcessProxy, private ProcessT
 
     void resetQuota(PAL::SessionID, CompletionHandler<void()>&&);
     void clearStorage(PAL::SessionID, CompletionHandler<void()>&&);
+#if PLATFORM(IOS_FAMILY)
+    void setBackupExclusionPeriodForTesting(PAL::SessionID, Seconds, CompletionHandler<void()>&&);
+#endif
 
     void resourceLoadDidSendRequest(WebPageProxyIdentifier, ResourceLoadInfo&&, WebCore::ResourceRequest&&, std::optional<IPC::FormDataReference>&&);
     void resourceLoadDidPerformHTTPRedirection(WebPageProxyIdentifier, ResourceLoadInfo&&, WebCore::ResourceResponse&&, WebCore::ResourceRequest&&);

Source/WebKit/UIProcess/Network/NetworkProcessProxyCocoa.mm

@@ -115,6 +115,11 @@
     [[NSNotificationCenter defaultCenter] removeObserver:m_foregroundObserver.get()];
 }
 
+void NetworkProcessProxy::setBackupExclusionPeriodForTesting(PAL::SessionID sessionID, Seconds period, CompletionHandler<void()>&& completionHandler)
+{
+    sendWithAsyncReply(Messages::NetworkProcess::SetBackupExclusionPeriodForTesting(sessionID, period), WTFMove(completionHandler));
+}
+
 #endif
 
 }

Source/WebKit/UIProcess/WebsiteData/Cocoa/WebsiteDataStoreCocoa.mm

@@ -705,6 +705,11 @@ static NavigatingToAppBoundDomain schemeOrDomainIsAppBound(const String& host, c
     return stringByResolvingSymlinksInPath(path);
 }
 
+void WebsiteDataStore::setBackupExclusionPeriodForTesting(Seconds period, CompletionHandler<void()>&& completionHandler)
+{
+    networkProcess().setBackupExclusionPeriodForTesting(m_sessionID, period, WTFMove(completionHandler));
+}
+
 #endif
 
 }

Source/WebKit/UIProcess/WebsiteData/WebsiteDataStore.h

@@ -363,6 +363,9 @@ class WebsiteDataStore : public API::ObjectImpl<API::Object::Type::WebsiteDataSt
 
     void resetQuota(CompletionHandler<void()>&&);
     void clearStorage(CompletionHandler<void()>&&);
+#if PLATFORM(IOS_FAMILY)
+    void setBackupExclusionPeriodForTesting(Seconds, CompletionHandler<void()>&&);
+#endif
 
 #if ENABLE(APP_BOUND_DOMAINS)
     void hasAppBoundSession(CompletionHandler<void(bool)>&&) const;