diff --git a/Package.swift b/Package.swift index 1a94aa5d..73b52f4e 100644 --- a/Package.swift +++ b/Package.swift @@ -21,7 +21,7 @@ let package = Package( url: "https://github.com/claucambra/NextcloudCapabilitiesKit.git", .upToNextMajor(from: "2.3.0") ), - .package(url: "https://github.com/nextcloud/NextcloudKit", from: "5.0.4"), + .package(url: "https://github.com/nextcloud/NextcloudKit", from: "6.0.9"), .package(url: "https://github.com/realm/realm-swift.git", exact: "20.0.1"), .package(url: "https://github.com/apple/swift-nio.git", from: "2.0.0") ], diff --git a/Sources/NextcloudFileProviderKit/Database/FilesDatabaseManager.swift b/Sources/NextcloudFileProviderKit/Database/FilesDatabaseManager.swift index 3bbb34f4..fe1ac3e3 100644 --- a/Sources/NextcloudFileProviderKit/Database/FilesDatabaseManager.swift +++ b/Sources/NextcloudFileProviderKit/Database/FilesDatabaseManager.swift @@ -250,7 +250,6 @@ public final class FilesDatabaseManager: Sendable { private func processItemMetadatasToUpdate( existingMetadatas: Results, updatedMetadatas: [SendableItemMetadata], - updateDirectoryEtags: Bool, keepExistingDownloadState: Bool ) -> ( newMetadatas: [SendableItemMetadata], @@ -268,16 +267,11 @@ public final class FilesDatabaseManager: Sendable { if existingMetadata.status == Status.normal.rawValue, !existingMetadata.isInSameDatabaseStoreableRemoteState(updatedMetadata) { - if updatedMetadata.directory { - if updatedMetadata.serverUrl != existingMetadata.serverUrl - || updatedMetadata.fileName != existingMetadata.fileName - { - directoriesNeedingRename.append(updatedMetadata) - updatedMetadata.etag = "" // Renaming doesn't change the etag so reset - - } else if !updateDirectoryEtags { - updatedMetadata.etag = existingMetadata.etag - } + if updatedMetadata.directory, + updatedMetadata.serverUrl != existingMetadata.serverUrl || + updatedMetadata.fileName != existingMetadata.fileName + { + directoriesNeedingRename.append(updatedMetadata) } if keepExistingDownloadState { @@ -305,11 +299,7 @@ public final class FilesDatabaseManager: Sendable { ) } - } else { // This is a new metadata - if !updateDirectoryEtags, updatedMetadata.directory { - updatedMetadata.etag = "" - } - + } else { // This is a new metadata returningNewMetadatas.append(updatedMetadata) Self.logger.debug( @@ -332,7 +322,6 @@ public final class FilesDatabaseManager: Sendable { account: String, serverUrl: String, updatedMetadatas: [SendableItemMetadata], - updateDirectoryEtags: Bool, keepExistingDownloadState: Bool ) -> ( newMetadatas: [SendableItemMetadata]?, @@ -360,7 +349,6 @@ public final class FilesDatabaseManager: Sendable { let metadatasToChange = processItemMetadatasToUpdate( existingMetadatas: existingMetadatas, updatedMetadatas: updatedMetadatas, - updateDirectoryEtags: updateDirectoryEtags, keepExistingDownloadState: keepExistingDownloadState ) @@ -591,7 +579,7 @@ public final class FilesDatabaseManager: Sendable { return parentItemIdentifier } - let (metadatas, _, _, _, error) = await Enumerator.readServerUrl( + let (metadatas, _, _, _, _, error) = await Enumerator.readServerUrl( metadata.serverUrl, account: account, remoteInterface: remoteInterface, diff --git a/Sources/NextcloudFileProviderKit/Enumeration/Enumerator+SyncEngine.swift b/Sources/NextcloudFileProviderKit/Enumeration/Enumerator+SyncEngine.swift index a54abe80..083e842a 100644 --- a/Sources/NextcloudFileProviderKit/Enumeration/Enumerator+SyncEngine.swift +++ b/Sources/NextcloudFileProviderKit/Enumeration/Enumerator+SyncEngine.swift @@ -99,27 +99,26 @@ extension Enumerator { Self.logger.debug("About to read: \(itemServerUrl, privacy: .public)") let ( - metadatas, newMetadatas, updatedMetadatas, deletedMetadatas, readError + metadatas, newMetadatas, updatedMetadatas, deletedMetadatas, _, readError ) = await Self.readServerUrl( itemServerUrl, account: account, remoteInterface: remoteInterface, dbManager: dbManager, domain: domain, - enumeratedItemIdentifier: enumeratedItemIdentifier, - stopAtMatchingEtags: scanChangesOnly + enumeratedItemIdentifier: enumeratedItemIdentifier ) if let readError, readError != .success { // Is the error is that we have found matching etags on this item, then ignore it // if we are doing a full rescan - if readError.isNoChangesError && scanChangesOnly { + if readError.isNoChangesError, scanChangesOnly { Self.logger.info("No changes in \(self.serverUrl) and only scanning changes.") } else { Self.logger.error( """ Finishing enumeration of changes at \(itemServerUrl, privacy: .public) - with \(readError.errorDescription, privacy: .public) + with \(readError.errorDescription, privacy: .public) """ ) @@ -127,7 +126,7 @@ extension Enumerator { Self.logger.info( """ 404 error means item no longer exists. - Deleting metadata and reporting as deletion without error + Deleting metadata and reporting as deletion without error. """ ) @@ -212,9 +211,7 @@ extension Enumerator { var childDirectoriesToScan: [SendableItemMetadata] = [] var candidateMetadatas: [SendableItemMetadata] - if scanChangesOnly, fastEnumeration { - candidateMetadatas = allUpdatedMetadatas - } else if scanChangesOnly { + if scanChangesOnly { candidateMetadatas = allUpdatedMetadatas + allNewMetadatas } else { candidateMetadatas = allMetadatas @@ -225,7 +222,7 @@ extension Enumerator { } Self.logger.debug( - "Candidate metadatas for further scan: \(candidateMetadatas, privacy: .public)" + "Candidate metadatas for further scan: \(childDirectoriesToScan, privacy: .public)" ) if childDirectoriesToScan.isEmpty { @@ -239,10 +236,11 @@ extension Enumerator { } for childDirectory in childDirectoriesToScan { + let childDirectoryUrl = childDirectory.serverUrl + "/" + childDirectory.fileName Self.logger.debug( """ - About to recursively scan: \(childDirectory.urlBase, privacy: .public) - with etag: \(childDirectory.etag, privacy: .public) + About to recursively scan: \(childDirectoryUrl, privacy: .public) + with etag: \(childDirectory.etag, privacy: .public) """ ) let childScanResult = await scanRecursively( @@ -266,11 +264,33 @@ extension Enumerator { ) } + static func handlePagedReadResults( + files: [NKFile], pageIndex: Int, dbManager: FilesDatabaseManager + ) -> (metadatas: [SendableItemMetadata]?, error: NKError?) { + // First PROPFIND contains the target item, but we do not want to report this in the + // retrieved metadatas (the enumeration observers don't expect you to enumerate the + // target item, hence why we always strip the target item out) + let startIndex = pageIndex > 0 ? 0 : 1 + if pageIndex == 0 { + guard let firstFile = files.first else { + return (nil, .invalidResponseError) + } + dbManager.addItemMetadata(firstFile.toItemMetadata()) + } + let metadatas = files[startIndex.. ( metadatas: [SendableItemMetadata]?, newMetadatas: [SendableItemMetadata]?, @@ -281,10 +301,15 @@ extension Enumerator { Self.logger.debug( """ Starting async conversion of NKFiles for serverUrl: \(serverUrl, privacy: .public) - for user: \(account.ncKitAccount, privacy: .public) + for user: \(account.ncKitAccount, privacy: .public) """ ) + if let pageIndex { + let (metadatas, error) = + handlePagedReadResults(files: files, pageIndex: pageIndex, dbManager: dbManager) + return (metadatas, nil, nil, nil, error) + } guard var (directoryMetadata, _, metadatas) = await files.toDirectoryReadMetadatas(account: account) @@ -294,8 +319,6 @@ extension Enumerator { } // STORE DATA FOR CURRENTLY SCANNED DIRECTORY - // We have now scanned this directory's contents, so update with etag in order to not check - // again if not needed unless it's the root container if serverUrl != account.davFilesUrl { if let existingMetadata = dbManager.itemMetadata(ocId: directoryMetadata.ocId) { directoryMetadata.downloaded = existingMetadata.downloaded @@ -303,16 +326,10 @@ extension Enumerator { dbManager.addItemMetadata(directoryMetadata) } - // Don't update the etags for folders as we haven't checked their contents. - // When we do a recursive check, if we update the etags now, we will think - // that our local copies are up to date -- instead, leave them as the old. - // They will get updated when they are the subject of a readServerUrl call. - // (See above) let changedMetadatas = dbManager.depth1ReadUpdateItemMetadatas( account: account.ncKitAccount, serverUrl: serverUrl, updatedMetadatas: metadatas, - updateDirectoryEtags: false, keepExistingDownloadState: true ) @@ -325,20 +342,34 @@ extension Enumerator { ) } + // READ THIS CAREFULLY. + // + // This method supports paginated and non-paginated reads. Handled by the pageSettings argument. + // Paginated reads is used by enumerateItems, non-paginated reads is used by enumerateChanges. + // + // Paginated reads WILL NOT HANDLE REMOVAL OF REMOTELY DELETED ITEMS FROM THE LOCAL DATABASE. + // Paginated reads WILL ONLY REPORT THE FILES DISCOVERED LOCALLY. + // This means that if you decide to use this method to implement change enumeration, you will + // have to collect the full results of all the pages before proceeding with discovering what + // has changed relative to the state of the local database -- manually! + // + // Non-paginated reads will update the database with all of the discovered files and folders + // that have been found to be created, updated, and deleted. No extra work required. static func readServerUrl( _ serverUrl: String, + pageSettings: (page: NSFileProviderPage?, index: Int, size: Int)? = nil, account: Account, remoteInterface: RemoteInterface, dbManager: FilesDatabaseManager, domain: NSFileProviderDomain? = nil, enumeratedItemIdentifier: NSFileProviderItemIdentifier? = nil, - stopAtMatchingEtags: Bool = false, depth: EnumerateDepth = .targetAndDirectChildren ) async -> ( metadatas: [SendableItemMetadata]?, newMetadatas: [SendableItemMetadata]?, updatedMetadatas: [SendableItemMetadata]?, deletedMetadatas: [SendableItemMetadata]?, + nextPage: EnumeratorPageResponse?, readError: NKError? ) { let ncKitAccount = account.ncKitAccount @@ -346,22 +377,33 @@ extension Enumerator { Self.logger.debug( """ Starting to read serverUrl: \(serverUrl, privacy: .public) - for user: \(ncKitAccount, privacy: .public) - at depth \(depth.rawValue, privacy: .public). - username: \(account.username, privacy: .public), - password is empty: \(account.password == "" ? "EMPTY" : "NOT EMPTY"), - serverUrl: \(account.serverUrl, privacy: .public) + for user: \(ncKitAccount, privacy: .public) + at depth \(depth.rawValue, privacy: .public). + username: \(account.username, privacy: .public), + password is empty: \(account.password == "" ? "EMPTY" : "NOT EMPTY"), + serverUrl: \(account.serverUrl, privacy: .public) """ ) - let (_, files, _, error) = await remoteInterface.enumerate( + let options: NKRequestOptions + if let pageSettings { + options = .init( + page: pageSettings.page, + offset: pageSettings.index * pageSettings.size, + count: pageSettings.size + ) + } else { + options = .init() + } + + let (_, files, data, error) = await remoteInterface.enumerate( remotePath: serverUrl, depth: depth, showHiddenFiles: true, includeHiddenFiles: [], requestBody: nil, account: account, - options: .init(), + options: options, taskHandler: { task in if let domain, let enumeratedItemIdentifier { NSFileProviderManager(for: domain)?.register( @@ -380,61 +422,61 @@ extension Enumerator { did not complete successfully, error: \(error.errorDescription, privacy: .public) """ ) - return (nil, nil, nil, nil, error) + return (nil, nil, nil, nil, nil, error) } - guard let receivedFile = files.first else { + guard let data else { Self.logger.error( """ - Received no items from readFileOrFolder of \(serverUrl, privacy: .public), - not much we can do... + \(depth.rawValue, privacy: .public) depth read of url \(serverUrl, privacy: .public) + did not return data. """ ) - return (nil, nil, nil, nil, error) + return (nil, nil, nil, nil, nil, error) } - guard receivedFile.directory else { - Self.logger.debug( - """ - Read item is a file. Converting NKfile for serverUrl: \(serverUrl, privacy: .public) - for user: \(account.ncKitAccount, privacy: .public) - """ - ) - var metadata = receivedFile.toItemMetadata() - let existing = dbManager.itemMetadata(ocId: metadata.ocId) - let isNew = existing == nil - let newItems: [SendableItemMetadata] = isNew ? [metadata] : [] - let updatedItems: [SendableItemMetadata] = isNew ? [] : [metadata] - metadata.downloaded = existing?.downloaded == true - dbManager.addItemMetadata(metadata) - return ([metadata], newItems, updatedItems, nil, nil) - } + // This will be nil if the page settings were also nil, as the server will not give us the + // pagination-related headers. + let nextPage = EnumeratorPageResponse( + nkResponseData: data, index: (pageSettings?.index ?? 0) + 1 + ) - if stopAtMatchingEtags, - let dir = dbManager.itemMetadata(account: ncKitAccount, locatedAtRemoteUrl: serverUrl), - dir.etag != "", - dir.etag == receivedFile.etag - { - Self.logger.debug( + guard let receivedFile = files.first else { + Self.logger.error( """ - Read server url called with flag to stop enumerating at matching etags. - Returning and providing soft error. + Received no items from readFileOrFolder of \(serverUrl, privacy: .public), + not much we can do... """ ) + return (nil, nil, nil, nil, nextPage, error) + } - let description = "Fetched directory etag same as local copy. Ignoring child items." - let nkError = NKError( - errorCode: NKError.noChangesErrorCode, errorDescription: description - ) - // Return all database metadatas under the current serverUrl (including target) - let metadatas = - dbManager.itemMetadatas(account: ncKitAccount, underServerUrl: serverUrl) - return (metadatas, nil, nil, nil, nkError) + // Generally speaking a PROPFIND will provide the target of the PROPFIND as the first result + // That is NOT the case for paginated results with offsets + let isFollowUpPaginatedRequest = (pageSettings?.page != nil && pageSettings?.index ?? 0 > 0) + if !isFollowUpPaginatedRequest { + guard receivedFile.directory else { + Self.logger.debug( + """ + Read item is a file. + Converting NKfile for serverUrl: \(serverUrl, privacy: .public) + for user: \(account.ncKitAccount, privacy: .public) + """ + ) + var metadata = receivedFile.toItemMetadata() + let existing = dbManager.itemMetadata(ocId: metadata.ocId) + let isNew = existing == nil + let newItems: [SendableItemMetadata] = isNew ? [metadata] : [] + let updatedItems: [SendableItemMetadata] = isNew ? [] : [metadata] + metadata.downloaded = existing?.downloaded == true + dbManager.addItemMetadata(metadata) + return ([metadata], newItems, updatedItems, nil, nextPage, nil) + } } if depth == .target { if serverUrl == account.davFilesUrl { - return (nil, nil, nil, nil, nil) + return (nil, nil, nil, nil, nextPage, nil) } else { var metadata = receivedFile.toItemMetadata() let existing = dbManager.itemMetadata(ocId: metadata.ocId) @@ -445,19 +487,28 @@ extension Enumerator { metadata.downloaded = existing?.downloaded == true dbManager.addItemMetadata(metadata) - return ([metadata], newMetadatas, updatedMetadatas, nil, nil) + return ([metadata], newMetadatas, updatedMetadatas, nil, nextPage, nil) } - } else { + } else if depth == .targetAndDirectChildren { let ( allMetadatas, newMetadatas, updatedMetadatas, deletedMetadatas, readError ) = await handleDepth1ReadFileOrFolder( serverUrl: serverUrl, account: account, dbManager: dbManager, - files: files + files: files, + pageIndex: pageSettings?.index ) - return (allMetadatas, newMetadatas, updatedMetadatas, deletedMetadatas, readError) + return (allMetadatas, newMetadatas, updatedMetadatas, deletedMetadatas, nextPage, readError) + } else if let pageIndex = pageSettings?.index { + let (metadatas, error) = handlePagedReadResults( + files: files, pageIndex: pageIndex, dbManager: dbManager + ) + return (metadatas, nil, nil, nil, nextPage, error) + } else { + // Infinite depth unpaged reads are a bad idea + return (nil, nil, nil, nil, nil, .invalidResponseError) } } } diff --git a/Sources/NextcloudFileProviderKit/Enumeration/Enumerator.swift b/Sources/NextcloudFileProviderKit/Enumeration/Enumerator.swift index 05f98474..f24d8888 100644 --- a/Sources/NextcloudFileProviderKit/Enumeration/Enumerator.swift +++ b/Sources/NextcloudFileProviderKit/Enumeration/Enumerator.swift @@ -27,11 +27,11 @@ public class Enumerator: NSObject, NSFileProviderEnumerator { // TODO: actually use this in NCKit and server requests private let anchor = NSFileProviderSyncAnchor(Date().description.data(using: .utf8)!) - private static let maxItemsPerFileProviderPage = 100 + private let pageItemCount: Int + private var pageNum = 0 static let logger = Logger(subsystem: Logger.subsystem, category: "enumerator") let account: Account let remoteInterface: RemoteInterface - let fastEnumeration: Bool var serverUrl: String = "" var isInvalidated = false weak var listener: EnumerationListener? @@ -46,16 +46,16 @@ public class Enumerator: NSObject, NSFileProviderEnumerator { remoteInterface: RemoteInterface, dbManager: FilesDatabaseManager, domain: NSFileProviderDomain? = nil, - fastEnumeration: Bool = true, - listener: EnumerationListener? = nil + listener: EnumerationListener? = nil, + pageSize: Int = 100 ) { self.enumeratedItemIdentifier = enumeratedItemIdentifier self.remoteInterface = remoteInterface self.account = account self.dbManager = dbManager self.domain = domain - self.fastEnumeration = fastEnumeration self.listener = listener + self.pageItemCount = pageSize if Self.isSystemIdentifier(enumeratedItemIdentifier) { Self.logger.debug( @@ -200,12 +200,9 @@ public class Enumerator: NSObject, NSFileProviderEnumerator { return } - // Handle the working set as if it were the root container - // If we do a full server scan per the recommendations of the File Provider documentation, - // we will be stuck for a huge period of time without being able to access files as the - // entire server gets scanned. Instead, treat the working set as the root container here. - // Then, when we enumerate changes, we'll go through everything -- while we can still - // navigate a little bit in Finder, file picker, etc + if enumeratedItemIdentifier == .workingSet { + Self.logger.info("Upcoming enumeration is of working set.") + } guard serverUrl != "" else { Self.logger.error( @@ -222,92 +219,80 @@ public class Enumerator: NSObject, NSFileProviderEnumerator { return } - // TODO: Make better use of pagination and handle paging properly - if page == NSFileProviderPage.initialPageSortedByDate as NSFileProviderPage - || page == NSFileProviderPage.initialPageSortedByName as NSFileProviderPage - { - Self.logger.debug( - """ - Enumerating initial page for user: \(self.account.ncKitAccount, privacy: .public) + Self.logger.debug( + """ + Enumerating page: \(self.pageNum, privacy: .public) + for user: \(self.account.ncKitAccount, privacy: .public) with serverUrl: \(self.serverUrl, privacy: .public) - """ - ) + """ + ) - Task { - let (metadatas, _, _, _, readError) = await Self.readServerUrl( - serverUrl, - account: account, - remoteInterface: remoteInterface, - dbManager: dbManager - ) + Task { + // Do not pass in the NSFileProviderPage default pages, these are not valid Nextcloud + // pagination tokens + var providedPage: NSFileProviderPage? = nil + if page != NSFileProviderPage.initialPageSortedByName as NSFileProviderPage && + page != NSFileProviderPage.initialPageSortedByDate as NSFileProviderPage + { + providedPage = page + } + let depth: EnumerateDepth = enumeratedItemIdentifier == .workingSet ? + .targetAndAllChildren : .targetAndDirectChildren + let (metadatas, _, _, _, nextPage, readError) = await Self.readServerUrl( + serverUrl, + pageSettings: (page: providedPage, index: pageNum, size: pageItemCount), + account: account, + remoteInterface: remoteInterface, + dbManager: dbManager, + depth: depth + ) - guard readError == nil else { - Self.logger.error( - """ - "Finishing enumeration for: \(self.account.ncKitAccount, privacy: .public) + guard readError == nil else { + Self.logger.error( + """ + Finishing enumeration for page: \(self.pageNum, privacy: .public) + for: \(self.account.ncKitAccount, privacy: .public) with serverUrl: \(self.serverUrl, privacy: .public) with error \(readError!.errorDescription, privacy: .public) - """ - ) - - // TODO: Refactor for conciseness - let error = readError?.fileProviderError( - handlingNoSuchItemErrorUsingItemIdentifier: self.enumeratedItemIdentifier - ) ?? NSFileProviderError(.cannotSynchronize) - listener?.enumerationActionFailed(actionId: actionId, error: error) - observer.finishEnumeratingWithError(error) - return - } + """ + ) - guard let metadatas else { - Self.logger.error( - """ - Finishing enumeration for: \(self.account.ncKitAccount, privacy: .public) - with serverUrl: \(self.serverUrl, privacy: .public) - with invalid metadatas. - """ - ) - listener?.enumerationActionFailed( - actionId: actionId, error: NSFileProviderError(.cannotSynchronize) - ) - observer.finishEnumeratingWithError(NSFileProviderError(.cannotSynchronize)) - return - } + // TODO: Refactor for conciseness + let error = readError?.fileProviderError( + handlingNoSuchItemErrorUsingItemIdentifier: self.enumeratedItemIdentifier + ) ?? NSFileProviderError(.cannotSynchronize) + listener?.enumerationActionFailed(actionId: actionId, error: error) + observer.finishEnumeratingWithError(error) + return + } - Self.logger.info( + guard let metadatas else { + Self.logger.error( """ - Finished reading serverUrl: \(self.serverUrl, privacy: .public) - for user: \(self.account.ncKitAccount, privacy: .public). - Processed \(metadatas.count) metadatas + Finishing enumeration for: \(self.account.ncKitAccount, privacy: .public) + with serverUrl: \(self.serverUrl, privacy: .public) + with invalid metadatas. """ ) - - Self.completeEnumerationObserver( - observer, - account: account, - remoteInterface: remoteInterface, - dbManager: dbManager, - numPage: 1, - itemMetadatas: metadatas + listener?.enumerationActionFailed( + actionId: actionId, error: NSFileProviderError(.cannotSynchronize) ) - listener?.enumerationActionFinished(actionId: actionId) + observer.finishEnumeratingWithError(NSFileProviderError(.cannotSynchronize)) + return } - return - } + Self.logger.info( + """ + Finished reading page: \(self.pageNum, privacy: .public) + serverUrl: \(self.serverUrl, privacy: .public) + for user: \(self.account.ncKitAccount, privacy: .public). + Processed \(metadatas.count) metadatas + """ + ) - let numPage = Int(String(data: page.rawValue, encoding: .utf8)!)! - Self.logger.debug( - """ - Enumerating page \(numPage, privacy: .public) - for user: \(self.account.ncKitAccount, privacy: .public) - with serverUrl: \(self.serverUrl, privacy: .public) - """ - ) - // TODO: Handle paging properly - // Self.completeObserver(observer, ncKit: ncKit, numPage: numPage, itemMetadatas: nil) - listener?.enumerationActionFinished(actionId: actionId) - observer.finishEnumerating(upTo: nil) + completeEnumerationObserver(observer, nextPage: nextPage, itemMetadatas: metadatas) + listener?.enumerationActionFinished(actionId: actionId) + } } public func enumerateChanges( @@ -476,13 +461,12 @@ public class Enumerator: NSObject, NSFileProviderEnumerator { // TODO: Move to the sync engine extension Task { let ( - _, newMetadatas, updatedMetadatas, deletedMetadatas, readError + _, newMetadatas, updatedMetadatas, deletedMetadatas, _, readError ) = await Self.readServerUrl( serverUrl, account: account, remoteInterface: remoteInterface, - dbManager: dbManager, - stopAtMatchingEtags: true + dbManager: dbManager ) // If we get a 404 we might add more deleted metadatas @@ -606,12 +590,9 @@ public class Enumerator: NSObject, NSFileProviderEnumerator { // NSFileProviderPage("\(numPage)".data(using: .utf8)!) } - private static func completeEnumerationObserver( + private func completeEnumerationObserver( _ observer: NSFileProviderEnumerationObserver, - account: Account, - remoteInterface: RemoteInterface, - dbManager: FilesDatabaseManager, - numPage: Int, + nextPage: EnumeratorPageResponse?, itemMetadatas: [SendableItemMetadata], handleInvalidParent: Bool = true ) { @@ -624,18 +605,12 @@ public class Enumerator: NSObject, NSFileProviderEnumerator { Task { @MainActor in observer.didEnumerate(items) Self.logger.info("Did enumerate \(items.count) items") - - // TODO: Handle paging properly - /* - if items.count == maxItemsPerFileProviderPage { - let nextPage = numPage + 1 - let providerPage = NSFileProviderPage("\(nextPage)".data(using: .utf8)!) - observer.finishEnumerating(upTo: providerPage) - } else { - observer.finishEnumerating(upTo: nil) - } - */ - observer.finishEnumerating(upTo: fileProviderPageforNumPage(numPage)) + if let nextPage, let nextPageData = nextPage.token.data(using: .utf8) { + self.pageNum = nextPage.index + observer.finishEnumerating(upTo: NSFileProviderPage(nextPageData)) + } else { + observer.finishEnumerating(upTo: nil) + } } } catch let error as NSError { // This error can only mean a missing parent item identifier guard handleInvalidParent else { @@ -650,12 +625,9 @@ public class Enumerator: NSObject, NSFileProviderEnumerator { remoteInterface: remoteInterface, dbManager: dbManager ) - Self.completeEnumerationObserver( + completeEnumerationObserver( observer, - account: account, - remoteInterface: remoteInterface, - dbManager: dbManager, - numPage: numPage, + nextPage: nextPage, itemMetadatas: [metadata] + itemMetadatas, handleInvalidParent: false ) @@ -787,7 +759,7 @@ public class Enumerator: NSObject, NSFileProviderEnumerator { Self.logger.info( "Recovering from invalid parent identifier at \(urlToEnumerate, privacy: .public)" ) - let (metadatas, _, _, _, error) = await Enumerator.readServerUrl( + let (metadatas, _, _, _, _, error) = await Enumerator.readServerUrl( urlToEnumerate, account: account, remoteInterface: remoteInterface, diff --git a/Sources/NextcloudFileProviderKit/Enumeration/EnumeratorPageResponse.swift b/Sources/NextcloudFileProviderKit/Enumeration/EnumeratorPageResponse.swift new file mode 100644 index 00000000..5d560c92 --- /dev/null +++ b/Sources/NextcloudFileProviderKit/Enumeration/EnumeratorPageResponse.swift @@ -0,0 +1,36 @@ +// +// EnumeratorPageResponse.swift +// NextcloudFileProviderKit +// +// Created by Claudio Cambra on 16/5/25. +// + +import Alamofire +import Foundation + +struct EnumeratorPageResponse: Sendable { + let token: String // Required by server to serve the next page of items + let index: Int // Needed to calculate the offset for the next paginated request + let total: Int? // Total item count, provided in the first non-offset paginated response + + init?(nkResponseData: AFDataResponse?, index: Int) { + guard let headers = nkResponseData?.response?.allHeaderFields as? [String: String] else { + return nil + } + + let normalisedHeaders = + Dictionary(uniqueKeysWithValues: headers.map { ($0.key.lowercased(), $0.value) }) + print(normalisedHeaders) + guard Bool(normalisedHeaders["x-nc-paginate"]?.lowercased() ?? "false") == true, + let responsePaginateToken = normalisedHeaders["x-nc-paginate-token"] + else { return nil } + + self.index = index + token = responsePaginateToken + if let responsePaginateTotal = normalisedHeaders["x-nc-paginate-total"] { + total = Int(responsePaginateTotal) + } else { + total = nil + } + } +} diff --git a/Sources/NextcloudFileProviderKit/Extensions/NKRequestOptions+Extensions.swift b/Sources/NextcloudFileProviderKit/Extensions/NKRequestOptions+Extensions.swift new file mode 100644 index 00000000..7de70953 --- /dev/null +++ b/Sources/NextcloudFileProviderKit/Extensions/NKRequestOptions+Extensions.swift @@ -0,0 +1,24 @@ +// +// NKRequestOptions+Extensions.swift +// NextcloudFileProviderKit +// +// Created by Claudio Cambra on 16/5/25. +// + +import FileProvider +import NextcloudKit + +extension NKRequestOptions { + convenience init(page: NSFileProviderPage?, offset: Int? = nil, count: Int? = nil) { + var token: String? = nil + if let page { + token = String(data: page.rawValue, encoding: .utf8) + } + self.init( + paginate: true, + paginateToken: token, + paginateOffset: offset, + paginateCount: count + ) + } +} diff --git a/Sources/NextcloudFileProviderKit/Interface/NextcloudKit+RemoteInterface.swift b/Sources/NextcloudFileProviderKit/Interface/NextcloudKit+RemoteInterface.swift index f379a780..3c7d5e41 100644 --- a/Sources/NextcloudFileProviderKit/Interface/NextcloudKit+RemoteInterface.swift +++ b/Sources/NextcloudFileProviderKit/Interface/NextcloudKit+RemoteInterface.swift @@ -278,7 +278,7 @@ extension NextcloudKit: RemoteInterface { options: NKRequestOptions = .init(), taskHandler: @escaping (URLSessionTask) -> Void = { _ in } ) async -> ( - account: String, files: [NKFile], data: Data?, error: NKError + account: String, files: [NKFile], data: AFDataResponse?, error: NKError ) { return await withCheckedContinuation { continuation in readFileOrFolder( @@ -291,7 +291,7 @@ extension NextcloudKit: RemoteInterface { options: options, taskHandler: taskHandler ) { account, files, data, error in - continuation.resume(returning: (account, files ?? [], data?.data, error)) + continuation.resume(returning: (account, files ?? [], data, error)) } } } diff --git a/Sources/NextcloudFileProviderKit/Interface/RemoteInterface.swift b/Sources/NextcloudFileProviderKit/Interface/RemoteInterface.swift index 2491193a..9d315afa 100644 --- a/Sources/NextcloudFileProviderKit/Interface/RemoteInterface.swift +++ b/Sources/NextcloudFileProviderKit/Interface/RemoteInterface.swift @@ -116,7 +116,7 @@ public protocol RemoteInterface { account: Account, options: NKRequestOptions, taskHandler: @escaping (_ task: URLSessionTask) -> Void - ) async -> (account: String, files: [NKFile], data: Data?, error: NKError) + ) async -> (account: String, files: [NKFile], data: AFDataResponse?, error: NKError) func delete( remotePath: String, diff --git a/Sources/NextcloudFileProviderKit/Item/Item+Create.swift b/Sources/NextcloudFileProviderKit/Item/Item+Create.swift index e3b40d82..10fda884 100644 --- a/Sources/NextcloudFileProviderKit/Item/Item+Create.swift +++ b/Sources/NextcloudFileProviderKit/Item/Item+Create.swift @@ -368,7 +368,7 @@ public extension Item { for remoteDirectoryPath in remoteDirectoriesPaths { // After everything, check into what the final state is of each folder now Self.logger.debug("Reading bpi folder at: \(remoteDirectoryPath, privacy: .public)") - let (_, _, _, _, readError) = await Enumerator.readServerUrl( + let (_, _, _, _, _, readError) = await Enumerator.readServerUrl( remoteDirectoryPath, account: account, remoteInterface: remoteInterface, @@ -567,7 +567,7 @@ public extension Item { Fetching remote information, proceeding with creation of internal contents. """ ) - let (metadatas, _, _, _, readError) = await Enumerator.readServerUrl( + let (metadatas, _, _, _, _, readError) = await Enumerator.readServerUrl( newServerUrlFileName, account: account, remoteInterface: remoteInterface, diff --git a/Sources/NextcloudFileProviderKit/Item/Item+Fetch.swift b/Sources/NextcloudFileProviderKit/Item/Item+Fetch.swift index df11d995..46131901 100644 --- a/Sources/NextcloudFileProviderKit/Item/Item+Fetch.swift +++ b/Sources/NextcloudFileProviderKit/Item/Item+Fetch.swift @@ -28,7 +28,7 @@ public extension Item { var remoteDirectoryPaths = [directoryRemotePath] while !remoteDirectoryPaths.isEmpty { let remoteDirectoryPath = remoteDirectoryPaths.removeFirst() - let (metadatas, _, _, _, readError) = await Enumerator.readServerUrl( + let (metadatas, _, _, _, _, readError) = await Enumerator.readServerUrl( remoteDirectoryPath, account: account, remoteInterface: remoteInterface, diff --git a/Sources/NextcloudFileProviderKit/Item/Item+Modify.swift b/Sources/NextcloudFileProviderKit/Item/Item+Modify.swift index 87d3241b..f8309848 100644 --- a/Sources/NextcloudFileProviderKit/Item/Item+Modify.swift +++ b/Sources/NextcloudFileProviderKit/Item/Item+Modify.swift @@ -269,7 +269,7 @@ public extension Item { var directoriesToRead = [remotePath] while !directoriesToRead.isEmpty { let remoteDirectoryPath = directoriesToRead.removeFirst() - let (metadatas, _, _, _, readError) = await Enumerator.readServerUrl( + let (metadatas, _, _, _, _, readError) = await Enumerator.readServerUrl( remoteDirectoryPath, account: account, remoteInterface: remoteInterface, @@ -482,7 +482,7 @@ public extension Item { for remoteDirectoryPath in remoteDirectoriesPaths { // After everything, check into what the final state is of each folder now - let (_, _, _, _, readError) = await Enumerator.readServerUrl( + let (_, _, _, _, _, readError) = await Enumerator.readServerUrl( remoteDirectoryPath, account: account, remoteInterface: remoteInterface, diff --git a/Tests/Interface/MockEnumerationObserver.swift b/Tests/Interface/MockEnumerationObserver.swift index d9161f80..d24e70d3 100644 --- a/Tests/Interface/MockEnumerationObserver.swift +++ b/Tests/Interface/MockEnumerationObserver.swift @@ -10,9 +10,12 @@ import Foundation public class MockEnumerationObserver: NSObject, NSFileProviderEnumerationObserver { public var items: [NSFileProviderItem] = [] + public var observedPages: [NSFileProviderPage] = [] private var error: Error? private var isComplete = false + private var currentPageComplete = false private var enumerator: NSFileProviderEnumerator + private var page: NSFileProviderPage? = nil public init(enumerator: NSFileProviderEnumerator) { self.enumerator = enumerator @@ -23,22 +26,33 @@ public class MockEnumerationObserver: NSObject, NSFileProviderEnumerationObserve } public func finishEnumerating(upTo nextPage: NSFileProviderPage?) { - isComplete = true + page = nextPage + isComplete = page == nil + currentPageComplete = true } public func finishEnumeratingWithError(_ error: Error) { self.error = error isComplete = true + currentPageComplete = true } public func enumerateItems() async throws { - let startPage = NSFileProviderPage.initialPageSortedByDate as NSFileProviderPage - enumerator.enumerateItems(for: self, startingAt: startPage) - while !isComplete { - try await Task.sleep(nanoseconds: 1_000_000) - } - if let error { - throw error + isComplete = false + currentPageComplete = false + observedPages = [] + page = NSFileProviderPage.initialPageSortedByName as NSFileProviderPage + + while let page, !isComplete { + observedPages.append(page) + enumerator.enumerateItems(for: self, startingAt: page) + while !currentPageComplete { + try await Task.sleep(nanoseconds: 1_000_000) + } + if let error { + throw error + } + currentPageComplete = false } } } diff --git a/Tests/Interface/MockRemoteInterface.swift b/Tests/Interface/MockRemoteInterface.swift index ca848d79..4df8e5b3 100644 --- a/Tests/Interface/MockRemoteInterface.swift +++ b/Tests/Interface/MockRemoteInterface.swift @@ -565,10 +565,17 @@ public class MockRemoteInterface: RemoteInterface { public var rootTrashItem: MockRemoteItem? public var currentChunks: [String: [RemoteFileChunk]] = [:] public var completedChunkTransferSize: [String: Int64] = [:] + public var pagination: Bool + public var expectedEnumerationPaginationTokens: [String: String] = [:] - public init(rootItem: MockRemoteItem? = nil, rootTrashItem: MockRemoteItem? = nil) { + public init( + rootItem: MockRemoteItem? = nil, + rootTrashItem: MockRemoteItem? = nil, + pagination: Bool = false + ) { self.rootItem = rootItem self.rootTrashItem = rootTrashItem + self.pagination = pagination } func sanitisedPath(_ path: String, account: Account) -> String? { @@ -1024,23 +1031,70 @@ public class MockRemoteInterface: RemoteInterface { account: Account, options: NKRequestOptions = .init(), taskHandler: @escaping (URLSessionTask) -> Void = { _ in } - ) async -> (account: String, files: [NKFile], data: Data?, error: NKError) { + ) async -> (account: String, files: [NKFile], data: AFDataResponse?, error: NKError) { print("Enumerating \(remotePath)") guard let item = item(remotePath: remotePath, account: account) else { print("Item at \(remotePath) not found.") return (account.ncKitAccount, [], nil, .urlError) } + func generateResponse(itemCount: Int, finalPage: Bool) -> AFDataResponse? { + var responseHeaders: [String: String] = [:] + if pagination && options.paginate { + responseHeaders["X-NC-PAGINATE"] = "true" + if options.paginateToken == nil { + responseHeaders["X-NC-PAGINATE-TOTAL"] = String(itemCount) + } + if finalPage { + expectedEnumerationPaginationTokens.removeValue(forKey: account.ncKitAccount) + } else { + let token = UUID().uuidString + responseHeaders["X-NC-PAGINATE-TOKEN"] = token + expectedEnumerationPaginationTokens[account.ncKitAccount] = token + } + } + + return AFDataResponse( + request: nil, + response: HTTPURLResponse( + url: URL(string: account.davFilesUrl + remotePath)!, + statusCode: 200, + httpVersion: "HTTP/1.1", + headerFields: responseHeaders + ), + data: Data(), + metrics: nil, + serializationDuration: 0, + result: .success(Data()) + ) + } + + let itemCount = options.paginateCount ?? .max + let firstItem = options.paginateOffset ?? 0 + + func generateReturn(files: [NKFile]) -> ( + account: String, files: [NKFile], data: AFDataResponse?, error: NKError + ) { + if pagination && + options.paginate && + options.paginateToken != expectedEnumerationPaginationTokens[account.ncKitAccount] + { + return (account.ncKitAccount, [], nil, .invalidData) + } + let reachedEnd = firstItem + itemCount >= files.count + let lastItem = min(firstItem + itemCount, files.count) - 1 + let itemsPage = Array(files[firstItem...lastItem]) + let responseData = generateResponse(itemCount: files.count, finalPage: reachedEnd) + return (account.ncKitAccount, itemsPage, responseData, .success) + } + switch depth { case .target: - return (account.ncKitAccount, [item.toNKFile()], nil, .success) + let responseData = generateResponse(itemCount: 1, finalPage: true) + return (account.ncKitAccount, [item.toNKFile()], responseData, .success) case .targetAndDirectChildren: - return ( - account.ncKitAccount, - [item.toNKFile()] + item.children.map { $0.toNKFile() }, - nil, - .success - ) + let files = [item.toNKFile()] + item.children.map { $0.toNKFile() } + return generateReturn(files: files) case .targetAndAllChildren: var files = [NKFile]() var queue = [item] @@ -1052,7 +1106,7 @@ public class MockRemoteInterface: RemoteInterface { } queue = nextQueue } - return (account.ncKitAccount, files, nil, .success) + return generateReturn(files: files) } } diff --git a/Tests/NextcloudFileProviderKitTests/EnumeratorPageResponseTests.swift b/Tests/NextcloudFileProviderKitTests/EnumeratorPageResponseTests.swift new file mode 100644 index 00000000..4c90a4a9 --- /dev/null +++ b/Tests/NextcloudFileProviderKitTests/EnumeratorPageResponseTests.swift @@ -0,0 +1,200 @@ +// +// EnumeratorPageResponseTests.swift +// NextcloudFileProviderKit +// +// Created by Claudio Cambra on 16/5/25. +// + +import Alamofire +import Foundation +import Testing +@testable import NextcloudFileProviderKit + +@Suite struct EnumeratorPageResponseTests { + private func createMockAFDataResponse( + headers: [String: String]?, + statusCode: Int = 200, + data: Data? = Data() + ) -> AFDataResponse? { + guard let url = URL(string: "https://example.com") else { + print("Error: Failed to create URL in test helper.") + return nil + } + let httpResponse = HTTPURLResponse( + url: url, + statusCode: statusCode, + httpVersion: "HTTP/1.1", + headerFields: headers + ) + let result: Result = .success(data ?? Data()) + return AFDataResponse( + request: nil, + response: httpResponse, + data: data, + metrics: nil, + serializationDuration: 0, + result: result + ) + } + + // MARK: - Success Cases + + @Test("Init with valid headers and total succeeds") + func initWithValidHeadersAndTotal() { + let headers = [ + "X-NC-PAGINATE": "true", + "X-NC-PAGINATE-TOKEN": "nextToken123", + "X-NC-PAGINATE-TOTAL": "100" + ] + let mockResponse = createMockAFDataResponse(headers: headers) + let index = 0 + + let enumeratorResponse = EnumeratorPageResponse(nkResponseData: mockResponse, index: index) + + #expect(enumeratorResponse != nil, "Initialization should succeed with valid values.") + #expect(enumeratorResponse?.token == "nextToken123") + #expect(enumeratorResponse?.index == 0) + #expect(enumeratorResponse?.total == 100) + } + + @Test("Init with valid headers and missing total succeeds") + func initWithValidHeadersAndMissingTotal() { + let headers = ["X-NC-PAGINATE": "true", "X-NC-PAGINATE-TOKEN": "anotherToken456"] + let mockResponse = createMockAFDataResponse(headers: headers) + let index = 1 + + let enumeratorResponse = EnumeratorPageResponse(nkResponseData: mockResponse, index: index) + + #expect(enumeratorResponse != nil, "Initialization should succeed with valid values.") + #expect(enumeratorResponse?.token == "anotherToken456") + #expect(enumeratorResponse?.index == 1) + #expect(enumeratorResponse?.total == nil, "Total should be nil when the header is missing.") + } + + @Test("Init with case-insensitive header keys and 'TRUE' succeeds") + func initWithCaseInsensitiveHeaders() { + let headers = [ + "x-nc-paginate": "TRUE", // Lowercase key, uppercase value for boolean + "x-nc-paginate-token": "mixedCaseToken789", + "X-NC-PAGINATE-TOTAL": "50" + ] + let mockResponse = createMockAFDataResponse(headers: headers) + let index = 2 + + let enumeratorResponse = EnumeratorPageResponse(nkResponseData: mockResponse, index: index) + + #expect(enumeratorResponse != nil, "Init should succeed with case-insensitive headers.") + #expect(enumeratorResponse?.token == "mixedCaseToken789") + #expect(enumeratorResponse?.index == 2) + #expect(enumeratorResponse?.total == 50) + } + + @Test("Init with non-integer total value results in nil total") + func initWithNonIntegerTotal() { + let headers = [ + "X-NC-PAGINATE": "true", + "X-NC-PAGINATE-TOKEN": "tokenWithInvalidTotal", + "X-NC-PAGINATE-TOTAL": "not-an-integer" + ] + let mockResponse = createMockAFDataResponse(headers: headers) + let index = 3 + + let enumeratorResponse = EnumeratorPageResponse(nkResponseData: mockResponse, index: index) + + #expect(enumeratorResponse != nil, "Init should succeed even if total is not valid integer") + #expect(enumeratorResponse?.token == "tokenWithInvalidTotal") + #expect(enumeratorResponse?.index == 3) + #expect(enumeratorResponse?.total == nil, "Total should be nil if cannot be parsed as Int") + } + + // MARK: - Failure Cases + + @Test("Init with nil nkResponseData returns nil") + func initWithNilNkResponseData() { + let index = 0 + let enumeratorResponse = EnumeratorPageResponse(nkResponseData: nil, index: index) + #expect(enumeratorResponse == nil, "Initialization should fail if nkResponseData is nil.") + } + + @Test("Init with nil HTTPURLResponse returns nil") + func initWithNilHttpResponse() { + let afResponseWithNilHttp = AFDataResponse( + request: nil, + response: nil, // HTTPURLResponse is nil + data: Data(), + metrics: nil, + serializationDuration: 0, + result: .success(Data()) + ) + let index = 0 + let enumeratorResponse = + EnumeratorPageResponse(nkResponseData: afResponseWithNilHttp, index: index) + #expect(enumeratorResponse == nil, "Initialization should fail if HTTPURLResponse is nil.") + } + + @Test("Init with empty headers returns nil") + func initWithEmptyHeaders() { + let mockResponse = createMockAFDataResponse(headers: [:]) + let index = 0 + + let enumeratorResponse = EnumeratorPageResponse(nkResponseData: mockResponse, index: index) + #expect(enumeratorResponse == nil, "Initialization should fail if required headers empty") + } + + @Test("Init without X-NC-PAGINATE header returns nil") + func initWithoutPaginateHeader() { + let headers = ["X-NC-PAGINATE-TOKEN": "someToken"] + let mockResponse = createMockAFDataResponse(headers: headers) + let index = 0 + + let enumeratorResponse = EnumeratorPageResponse(nkResponseData: mockResponse, index: index) + #expect(enumeratorResponse == nil, "Initialization should fail if PAGINATE header missing.") + } + + @Test("Init with X-NC-PAGINATE header set to 'false' returns nil") + func initWithPaginateHeaderFalse() { + let headers = [ + "X-NC-PAGINATE": "false", + "X-NC-PAGINATE-TOKEN": "someToken" + ] + let mockResponse = createMockAFDataResponse(headers: headers) + let index = 0 + + let enumeratorResponse = EnumeratorPageResponse(nkResponseData: mockResponse, index: index) + #expect(enumeratorResponse == nil, "Initialization should fail if PAGINATE header is false") + } + + @Test("Init with X-NC-PAGINATE header not a valid 'true' string returns nil") + func initWithPaginateHeaderNotTrueString() { + let headers = ["X-NC-PAGINATE": "false", "X-NC-PAGINATE-TOKEN": "someToken"] + let mockResponse = createMockAFDataResponse(headers: headers) + let index = 0 + + let enumeratorResponse = EnumeratorPageResponse(nkResponseData: mockResponse, index: index) + #expect(enumeratorResponse == nil, "Initialization should fail if PAGINATE header not true") + } + + @Test("Init without X-NC-PAGINATE-TOKEN header returns nil") + func initWithoutPaginateTokenHeader() { + let headers = ["X-NC-PAGINATE": "true"] + let mockResponse = createMockAFDataResponse(headers: headers) + let index = 0 + + let enumeratorResponse = EnumeratorPageResponse(nkResponseData: mockResponse, index: index) + #expect(enumeratorResponse == nil, "Initialization should fail if TOKEN header is missing.") + } + + @Test("Init with empty X-NC-PAGINATE-TOKEN header succeeds with empty token") + func initWithEmptyPaginateToken() { + // The current implementation allows an empty token if the header key exists. + let headers = ["X-NC-PAGINATE": "true", "X-NC-PAGINATE-TOKEN": ""] + let mockResponse = createMockAFDataResponse(headers: headers) + let index = 0 + + let enumeratorResponse = EnumeratorPageResponse(nkResponseData: mockResponse, index: index) + #expect(enumeratorResponse != nil, "Initialization should succeed with empty token string.") + #expect(enumeratorResponse?.token == "", "Token should be an empty string.") + #expect(enumeratorResponse?.index == 0) + #expect(enumeratorResponse?.total == nil) + } +} diff --git a/Tests/NextcloudFileProviderKitTests/EnumeratorTests.swift b/Tests/NextcloudFileProviderKitTests/EnumeratorTests.swift index 554ccfd5..38f3b305 100644 --- a/Tests/NextcloudFileProviderKitTests/EnumeratorTests.swift +++ b/Tests/NextcloudFileProviderKitTests/EnumeratorTests.swift @@ -154,9 +154,8 @@ final class EnumeratorTests: XCTestCase { Int(remoteFolder.modificationDate.timeIntervalSince1970) ) - // Important to keep in mind. Default behaviour is fast enumeration, not deep enumeration let dbFolder = try XCTUnwrap(Self.dbManager.itemMetadata(ocId: remoteFolder.identifier)) - XCTAssertEqual(dbFolder.etag, "") // Folder is not visited yet, should not have etag + XCTAssertEqual(dbFolder.etag, remoteFolder.versionIdentifier) XCTAssertEqual(dbFolder.fileName, remoteFolder.name) XCTAssertEqual(dbFolder.fileNameView, remoteFolder.name) XCTAssertEqual(dbFolder.serverUrl + "/" + dbFolder.fileName, remoteFolder.remotePath) @@ -196,7 +195,7 @@ final class EnumeratorTests: XCTestCase { ) let observer = MockEnumerationObserver(enumerator: enumerator) try await observer.enumerateItems() - XCTAssertEqual(observer.items.count, 1) // Should only get the folder in root + XCTAssertEqual(observer.items.count, 3) let retrievedFolderItem = try XCTUnwrap(observer.items.first) XCTAssertEqual(retrievedFolderItem.itemIdentifier.rawValue, remoteFolder.identifier) @@ -211,10 +210,10 @@ final class EnumeratorTests: XCTestCase { // Ensure the newly discovered folder has no etag let dbFolder = try XCTUnwrap(Self.dbManager.itemMetadata(ocId: remoteFolder.identifier)) - XCTAssertTrue(dbFolder.etag.isEmpty) + XCTAssertEqual(dbFolder.etag, remoteFolder.versionIdentifier) } - func testWorkingSetFastChangeEnumeration() async throws { + func testWorkingSetChangeEnumeration() async throws { let db = Self.dbManager.ncDatabase() // Strong ref for in memory test db debugPrint(db) let remoteInterface = MockRemoteInterface(rootItem: rootItem) @@ -227,52 +226,6 @@ final class EnumeratorTests: XCTestCase { ) let observer = MockChangeObserver(enumerator: enumerator) try await observer.enumerateChanges() - XCTAssertEqual(observer.changedItems.count, 1) // Should only get the folder in root - - let retrievedFolderItem = try XCTUnwrap(observer.changedItems.first) - XCTAssertEqual(retrievedFolderItem.itemIdentifier.rawValue, remoteFolder.identifier) - XCTAssertEqual(retrievedFolderItem.filename, remoteFolder.name) - XCTAssertEqual(retrievedFolderItem.parentItemIdentifier.rawValue, rootItem.identifier) - XCTAssertEqual(retrievedFolderItem.creationDate, remoteFolder.creationDate) - XCTAssertEqual( - Int(retrievedFolderItem.contentModificationDate??.timeIntervalSince1970 ?? 0), - Int(remoteFolder.modificationDate.timeIntervalSince1970) - ) - XCTAssertEqual(retrievedFolderItem.isUploaded, true) - - // Ensure the newly discovered folder has no etag - var dbFolder = try XCTUnwrap(Self.dbManager.itemMetadata(ocId: remoteFolder.identifier)) - XCTAssertTrue(dbFolder.etag.isEmpty) - - // Having an etag marks a folder as visited. - // We should get the two remaining files now, as the etag does not match the server but is - // present, marking the folder as explored - dbFolder.etag = "Not server etag" - Self.dbManager.addItemMetadata(dbFolder) - - let newObserver = MockChangeObserver(enumerator: enumerator) - try await newObserver.enumerateChanges() - XCTAssertEqual(newObserver.changedItems.count, 3) - - let newNewObsever = MockChangeObserver(enumerator: enumerator) - try await newNewObsever.enumerateChanges() - XCTAssertEqual(newNewObsever.changedItems.count, 0) - } - - func testWorkingSetSlowChangeEnumeration() async throws { - let db = Self.dbManager.ncDatabase() // Strong ref for in memory test db - debugPrint(db) - let remoteInterface = MockRemoteInterface(rootItem: rootItem) - - let enumerator = Enumerator( - enumeratedItemIdentifier: .workingSet, - account: Self.account, - remoteInterface: remoteInterface, - dbManager: Self.dbManager, - fastEnumeration: false - ) - let observer = MockChangeObserver(enumerator: enumerator) - try await observer.enumerateChanges() XCTAssertEqual(observer.changedItems.count, 3) // Should get all items let retrievedFolderItem = try XCTUnwrap(observer.changedItems.first) @@ -753,7 +706,6 @@ final class EnumeratorTests: XCTestCase { ) let observer = MockEnumerationObserver(enumerator: enumerator) try await observer.enumerateItems() - XCTAssertEqual(observer.items.count, 1) // Should only get the folder in root // Check enumeration actions XCTAssertEqual(listener.startActions.count, 1) @@ -992,4 +944,50 @@ final class EnumeratorTests: XCTestCase { Int(remoteItemA.modificationDate.timeIntervalSince1970) ) } + + func testFolderPaginatedEnumeration() async throws { + remoteFolder.children = [] + for i in 0...20 { + let childItem = MockRemoteItem( + identifier: "folderChild\(i)", + name: "folderChild\(i).txt", + remotePath: Self.account.davFilesUrl + "folder/folderChild\(i).txt", + account: Self.account.ncKitAccount, + username: Self.account.username, + userId: Self.account.id, + serverUrl: Self.account.serverUrl + ) + childItem.parent = remoteFolder + remoteFolder.children.append(childItem) + } + + let db = Self.dbManager.ncDatabase() // Strong ref for in memory test db + debugPrint(db) + let remoteInterface = MockRemoteInterface(rootItem: rootItem, pagination: true) + + let oldEtag = "OLD" + var folderMetadata = remoteFolder.toItemMetadata(account: Self.account) + folderMetadata.etag = oldEtag + + Self.dbManager.addItemMetadata(folderMetadata) + XCTAssertNotNil(Self.dbManager.itemMetadata(ocId: remoteFolder.identifier)) + + let enumerator = Enumerator( + enumeratedItemIdentifier: .init(remoteFolder.identifier), + account: Self.account, + remoteInterface: remoteInterface, + dbManager: Self.dbManager, + pageSize: 5 + ) + let observer = MockEnumerationObserver(enumerator: enumerator) + try await observer.enumerateItems() + XCTAssertEqual(observer.items.count, 21) + + for item in observer.items { + XCTAssertNotNil(Self.dbManager.itemMetadata(ocId: item.itemIdentifier.rawValue)) + } + + XCTAssertEqual(observer.observedPages.first, NSFileProviderPage.initialPageSortedByName as NSFileProviderPage) + XCTAssertEqual(observer.observedPages.count, 5) + } } diff --git a/Tests/NextcloudFileProviderKitTests/FilesDatabaseManagerTests.swift b/Tests/NextcloudFileProviderKitTests/FilesDatabaseManagerTests.swift index 3243f038..b04d73ba 100644 --- a/Tests/NextcloudFileProviderKitTests/FilesDatabaseManagerTests.swift +++ b/Tests/NextcloudFileProviderKitTests/FilesDatabaseManagerTests.swift @@ -78,7 +78,6 @@ final class FilesDatabaseManagerTests: XCTestCase { account: account.ncKitAccount, serverUrl: account.davFilesUrl, updatedMetadatas: [metadata], - updateDirectoryEtags: true, keepExistingDownloadState: true ) @@ -92,7 +91,6 @@ final class FilesDatabaseManagerTests: XCTestCase { account: account.ncKitAccount, serverUrl: account.davFilesUrl, updatedMetadatas: [metadata], - updateDirectoryEtags: true, keepExistingDownloadState: true ) @@ -109,7 +107,6 @@ final class FilesDatabaseManagerTests: XCTestCase { account: account.ncKitAccount, serverUrl: account.davFilesUrl, updatedMetadatas: [metadata], - updateDirectoryEtags: true, keepExistingDownloadState: true ) @@ -152,7 +149,6 @@ final class FilesDatabaseManagerTests: XCTestCase { account: account.ncKitAccount, serverUrl: account.davFilesUrl, updatedMetadatas: [renamedParent], - updateDirectoryEtags: true, keepExistingDownloadState: true ) @@ -189,7 +185,6 @@ final class FilesDatabaseManagerTests: XCTestCase { account: account.ncKitAccount, serverUrl: account.davFilesUrl, updatedMetadatas: [incoming], - updateDirectoryEtags: true, keepExistingDownloadState: true ) @@ -215,7 +210,6 @@ final class FilesDatabaseManagerTests: XCTestCase { account: account.ncKitAccount, serverUrl: account.davFilesUrl, updatedMetadatas: [], - updateDirectoryEtags: true, keepExistingDownloadState: true ) @@ -340,7 +334,6 @@ final class FilesDatabaseManagerTests: XCTestCase { account: "TestAccount", serverUrl: "https://example.com", updatedMetadatas: updatedMetadatas.map { SendableItemMetadata(value: $0) }, - updateDirectoryEtags: true, keepExistingDownloadState: true ) @@ -385,7 +378,6 @@ final class FilesDatabaseManagerTests: XCTestCase { account: "TestAccount", serverUrl: "https://example.com", updatedMetadatas: [updatedMetadata, newMetadata], - updateDirectoryEtags: true, keepExistingDownloadState: true ) @@ -442,7 +434,6 @@ final class FilesDatabaseManagerTests: XCTestCase { account: testAccount, serverUrl: testServerUrl, updatedMetadatas: updatedMetadatasFromServer, - updateDirectoryEtags: true, // Value doesn't strictly matter for this test logic keepExistingDownloadState: true // Value doesn't strictly matter for this test logic ) diff --git a/Tests/NextcloudFileProviderKitTests/RemoteInterfaceTests.swift b/Tests/NextcloudFileProviderKitTests/RemoteInterfaceTests.swift index dcd67511..e7fdae9d 100644 --- a/Tests/NextcloudFileProviderKitTests/RemoteInterfaceTests.swift +++ b/Tests/NextcloudFileProviderKitTests/RemoteInterfaceTests.swift @@ -107,7 +107,7 @@ fileprivate struct TestableRemoteInterface: RemoteInterface { account: Account, options: NKRequestOptions, taskHandler: @escaping (URLSessionTask) -> Void - ) async -> (account: String, files: [NKFile], data: Data?, error: NKError) { + ) async -> (account: String, files: [NKFile], data: AFDataResponse?, error: NKError) { ("", [], nil, .invalidResponseError) }