Process intermediates in chunks for CoreDataBatchImportOperation

PeakCoreData.xcodeproj/project.pbxproj

@@ -7,6 +7,7 @@
 	objects = {
 
 /* Begin PBXBuildFile section */
+		4C9978B207CECBCFAEC9305D /* libPods-PeakCoreDataTests.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 2150AB7DFE4EED6D3ABFBCEC /* libPods-PeakCoreDataTests.a */; };
 		8D05BE6D2051518B00DE93AC /* FetchedDataProvider.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8D05BE6C2051518B00DE93AC /* FetchedDataProvider.swift */; };
 		8D159AFE205C001A004D693A /* AnotherEntity.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8D159AFD205C001A004D693A /* AnotherEntity.swift */; };
 		8D159B00205C01EE004D693A /* ContextDidSaveNotification.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8D159AFF205C01EE004D693A /* ContextDidSaveNotification.swift */; };
@@ -28,7 +29,7 @@
 		8DFAC10F1F7918420015EDC0 /* CoreDataSingleImportOperation.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8DFAC10E1F7918420015EDC0 /* CoreDataSingleImportOperation.swift */; };
 		8DFAC1111F79274B0015EDC0 /* Changeset.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8DFAC1101F79274B0015EDC0 /* Changeset.swift */; };
 		8DFD6D66205A82F6008CFD28 /* ManagedObjectObserver.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8DFD6D65205A82F6008CFD28 /* ManagedObjectObserver.swift */; };
-		97AA2B667908946878294909 /* Pods_PeakCoreData.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = CBEEAD29A8BB44890968C40B /* Pods_PeakCoreData.framework */; };
+		908EE9B74A9B51E3D94C7BD6 /* libPods-PeakCoreData.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 4AA0068DBA5242B3EF613E08 /* libPods-PeakCoreData.a */; };
 		CB13D8DE2058351C008B3678 /* FetchedCollection.swift in Sources */ = {isa = PBXBuildFile; fileRef = CB13D8DD2058351C008B3678 /* FetchedCollection.swift */; };
 		CB13D8E120583703008B3678 /* FetchedCollectionTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = CB13D8DF20583703008B3678 /* FetchedCollectionTests.swift */; };
 		CBF972A51E5DEE6D00F37801 /* CoreDataOperation.swift in Sources */ = {isa = PBXBuildFile; fileRef = CBF972A41E5DEE6D00F37801 /* CoreDataOperation.swift */; };
@@ -37,7 +38,6 @@
 		CBF972B51E5DEEFC00F37801 /* PersistentContainerSettable.swift in Sources */ = {isa = PBXBuildFile; fileRef = CBF972B41E5DEEFC00F37801 /* PersistentContainerSettable.swift */; };
 		CBF972BB1E5DEF2100F37801 /* ManagedObjectUpdatable.swift in Sources */ = {isa = PBXBuildFile; fileRef = CBF972BA1E5DEF2100F37801 /* ManagedObjectUpdatable.swift */; };
 		CBF972BE1E5DEF2900F37801 /* CoreDataBatchImportOperation.swift in Sources */ = {isa = PBXBuildFile; fileRef = CBF972BC1E5DEF2900F37801 /* CoreDataBatchImportOperation.swift */; };
-		F0B99C6E8634D38F3C4FE4F1 /* Pods_PeakCoreDataTests.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = E72B09366538DEA59EAFFC84 /* Pods_PeakCoreDataTests.framework */; };
 /* End PBXBuildFile section */
 
 /* Begin PBXContainerItemProxy section */
@@ -51,7 +51,9 @@
 /* End PBXContainerItemProxy section */
 
 /* Begin PBXFileReference section */
+		2150AB7DFE4EED6D3ABFBCEC /* libPods-PeakCoreDataTests.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = "libPods-PeakCoreDataTests.a"; sourceTree = BUILT_PRODUCTS_DIR; };
 		2922F3C57EE33341E8C70A53 /* Pods-PeakCoreDataTests.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-PeakCoreDataTests.debug.xcconfig"; path = "Pods/Target Support Files/Pods-PeakCoreDataTests/Pods-PeakCoreDataTests.debug.xcconfig"; sourceTree = "<group>"; };
+		4AA0068DBA5242B3EF613E08 /* libPods-PeakCoreData.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = "libPods-PeakCoreData.a"; sourceTree = BUILT_PRODUCTS_DIR; };
 		694BDD5F23B0917B776ABDE8 /* Pods-PeakCoreData.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-PeakCoreData.release.xcconfig"; path = "Pods/Target Support Files/Pods-PeakCoreData/Pods-PeakCoreData.release.xcconfig"; sourceTree = "<group>"; };
 		8D05BE6C2051518B00DE93AC /* FetchedDataProvider.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FetchedDataProvider.swift; sourceTree = "<group>"; };
 		8D159AFD205C001A004D693A /* AnotherEntity.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AnotherEntity.swift; sourceTree = "<group>"; };
@@ -80,23 +82,21 @@
 		C0A3AEC430E4B051590F505B /* Pods-PeakCoreDataTests.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-PeakCoreDataTests.release.xcconfig"; path = "Pods/Target Support Files/Pods-PeakCoreDataTests/Pods-PeakCoreDataTests.release.xcconfig"; sourceTree = "<group>"; };
 		CB13D8DD2058351C008B3678 /* FetchedCollection.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = FetchedCollection.swift; sourceTree = "<group>"; };
 		CB13D8DF20583703008B3678 /* FetchedCollectionTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = FetchedCollectionTests.swift; sourceTree = "<group>"; };
-		CBEEAD29A8BB44890968C40B /* Pods_PeakCoreData.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Pods_PeakCoreData.framework; sourceTree = BUILT_PRODUCTS_DIR; };
 		CBF972A41E5DEE6D00F37801 /* CoreDataOperation.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = CoreDataOperation.swift; sourceTree = "<group>"; };
 		CBF972B01E5DEE8700F37801 /* ManagedObjectType.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ManagedObjectType.swift; sourceTree = "<group>"; };
 		CBF972B11E5DEE8700F37801 /* UniqueIdentifiable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = UniqueIdentifiable.swift; sourceTree = "<group>"; };
 		CBF972B41E5DEEFC00F37801 /* PersistentContainerSettable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PersistentContainerSettable.swift; sourceTree = "<group>"; };
 		CBF972BA1E5DEF2100F37801 /* ManagedObjectUpdatable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ManagedObjectUpdatable.swift; sourceTree = "<group>"; };
 		CBF972BC1E5DEF2900F37801 /* CoreDataBatchImportOperation.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = CoreDataBatchImportOperation.swift; sourceTree = "<group>"; };
 		DAC0D3C03C9FDAFE479F57F3 /* Pods-PeakCoreData.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-PeakCoreData.debug.xcconfig"; path = "Pods/Target Support Files/Pods-PeakCoreData/Pods-PeakCoreData.debug.xcconfig"; sourceTree = "<group>"; };
-		E72B09366538DEA59EAFFC84 /* Pods_PeakCoreDataTests.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Pods_PeakCoreDataTests.framework; sourceTree = BUILT_PRODUCTS_DIR; };
 /* End PBXFileReference section */
 
 /* Begin PBXFrameworksBuildPhase section */
 		8D9797D71DF8138100DAB75A /* Frameworks */ = {
 			isa = PBXFrameworksBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
-				97AA2B667908946878294909 /* Pods_PeakCoreData.framework in Frameworks */,
+				908EE9B74A9B51E3D94C7BD6 /* libPods-PeakCoreData.a in Frameworks */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};
@@ -105,7 +105,7 @@
 			buildActionMask = 2147483647;
 			files = (
 				8D9797E51DF8138100DAB75A /* PeakCoreData.framework in Frameworks */,
-				F0B99C6E8634D38F3C4FE4F1 /* Pods_PeakCoreDataTests.framework in Frameworks */,
+				4C9978B207CECBCFAEC9305D /* libPods-PeakCoreDataTests.a in Frameworks */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};
@@ -246,8 +246,8 @@
 		B6BED3CCDC8E0ACD5DB44FA6 /* Frameworks */ = {
 			isa = PBXGroup;
 			children = (
-				CBEEAD29A8BB44890968C40B /* Pods_PeakCoreData.framework */,
-				E72B09366538DEA59EAFFC84 /* Pods_PeakCoreDataTests.framework */,
+				4AA0068DBA5242B3EF613E08 /* libPods-PeakCoreData.a */,
+				2150AB7DFE4EED6D3ABFBCEC /* libPods-PeakCoreDataTests.a */,
 			);
 			name = Frameworks;
 			sourceTree = "<group>";
@@ -293,7 +293,6 @@
 				8D9797E01DF8138100DAB75A /* Sources */,
 				8D9797E11DF8138100DAB75A /* Frameworks */,
 				8D9797E21DF8138100DAB75A /* Resources */,
-				D2E8D86DAD84747A65F17DFF /* [CP] Embed Pods Frameworks */,
 			);
 			buildRules = (
 			);
@@ -383,26 +382,6 @@
 			shellScript = "diff \"${PODS_PODFILE_DIR_PATH}/Podfile.lock\" \"${PODS_ROOT}/Manifest.lock\" > /dev/null\nif [ $? != 0 ] ; then\n    # print error to STDERR\n    echo \"error: The sandbox is not in sync with the Podfile.lock. Run 'pod install' or update your CocoaPods installation.\" >&2\n    exit 1\nfi\n# This output is used by Xcode 'outputs' to avoid re-running this script phase.\necho \"SUCCESS\" > \"${SCRIPT_OUTPUT_FILE_0}\"\n";
 			showEnvVarsInLog = 0;
 		};
-		D2E8D86DAD84747A65F17DFF /* [CP] Embed Pods Frameworks */ = {
-			isa = PBXShellScriptBuildPhase;
-			buildActionMask = 2147483647;
-			files = (
-			);
-			inputPaths = (
-				"${SRCROOT}/Pods/Target Support Files/Pods-PeakCoreDataTests/Pods-PeakCoreDataTests-frameworks.sh",
-				"${BUILT_PRODUCTS_DIR}/PeakOperation/PeakOperation.framework",
-				"${BUILT_PRODUCTS_DIR}/PeakResult/PeakResult.framework",
-			);
-			name = "[CP] Embed Pods Frameworks";
-			outputPaths = (
-				"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/PeakOperation.framework",
-				"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/PeakResult.framework",
-			);
-			runOnlyForDeploymentPostprocessing = 0;
-			shellPath = /bin/sh;
-			shellScript = "\"${SRCROOT}/Pods/Target Support Files/Pods-PeakCoreDataTests/Pods-PeakCoreDataTests-frameworks.sh\"\n";
-			showEnvVarsInLog = 0;
-		};
 		F47EEA9569966FF7DEE02C6D /* [CP] Check Pods Manifest.lock */ = {
 			isa = PBXShellScriptBuildPhase;
 			buildActionMask = 2147483647;

PeakCoreData.xcodeproj/xcshareddata/xcbaselines/8D9797E31DF8138100DAB75A.xcbaseline/A3246E1C-1BC2-45A8-A67A-99C1C6B2E5D8.plist

@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>classNames</key>
+	<dict>
+		<key>OperationTests</key>
+		<dict>
+			<key>testChunkedBatchImportPerformance_BatchSize1000()</key>
+			<dict>
+				<key>com.apple.XCTPerformanceMetric_WallClockTime</key>
+				<dict>
+					<key>baselineAverage</key>
+					<real>2.7685</real>
+					<key>baselineIntegrationDisplayName</key>
+					<string>18 Feb 2019 at 13:35:54</string>
+				</dict>
+			</dict>
+			<key>testChunkedBatchImportPerformance_BatchSize200()</key>
+			<dict>
+				<key>com.apple.XCTPerformanceMetric_WallClockTime</key>
+				<dict>
+					<key>baselineAverage</key>
+					<real>3.6737</real>
+					<key>baselineIntegrationDisplayName</key>
+					<string>18 Feb 2019 at 13:35:54</string>
+				</dict>
+			</dict>
+			<key>testChunkedBatchImportPerformance_BatchSizeMax()</key>
+			<dict>
+				<key>com.apple.XCTPerformanceMetric_WallClockTime</key>
+				<dict>
+					<key>baselineAverage</key>
+					<real>2.438</real>
+					<key>baselineIntegrationDisplayName</key>
+					<string>18 Feb 2019 at 13:35:54</string>
+				</dict>
+			</dict>
+		</dict>
+	</dict>
+</dict>
+</plist>

PeakCoreData.xcodeproj/xcshareddata/xcbaselines/8D9797E31DF8138100DAB75A.xcbaseline/Info.plist

@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>runDestinationsByUUID</key>
+	<dict>
+		<key>A3246E1C-1BC2-45A8-A67A-99C1C6B2E5D8</key>
+		<dict>
+			<key>localComputer</key>
+			<dict>
+				<key>busSpeedInMHz</key>
+				<integer>100</integer>
+				<key>cpuCount</key>
+				<integer>1</integer>
+				<key>cpuKind</key>
+				<string>Intel Core i5</string>
+				<key>cpuSpeedInMHz</key>
+				<integer>3300</integer>
+				<key>logicalCPUCoresPerPackage</key>
+				<integer>4</integer>
+				<key>modelCode</key>
+				<string>iMac15,1</string>
+				<key>physicalCPUCoresPerPackage</key>
+				<integer>4</integer>
+				<key>platformIdentifier</key>
+				<string>com.apple.platform.macosx</string>
+			</dict>
+			<key>targetArchitecture</key>
+			<string>x86_64</string>
+			<key>targetDevice</key>
+			<dict>
+				<key>modelCode</key>
+				<string>iPhone11,8</string>
+				<key>platformIdentifier</key>
+				<string>com.apple.platform.iphonesimulator</string>
+			</dict>
+		</dict>
+	</dict>
+</dict>
+</plist>

PeakCoreData/Operations/CoreDataBatchImportOperation.swift

@@ -11,26 +11,44 @@ import CoreData
 import PeakOperation
 import PeakResult
 
+extension Array {
+    func chunked(into size: Int) -> [[Element]] {
+        return stride(from: 0, to: count, by: size).map {
+            Array(self[$0 ..< Swift.min($0 + size, count)])
+        }
+    }
+}
+
 open class CoreDataBatchImportOperation<Intermediate>: CoreDataChangesetOperation, ConsumesResult where
     Intermediate: ManagedObjectUpdatable & UniqueIdentifiable,
     Intermediate.ManagedObject: ManagedObjectType & UniqueIdentifiable
 {
+    typealias ManagedObject = Intermediate.ManagedObject
+
     public var input: Result<[Intermediate]> = Result { throw ResultError.noResult }
 
-    typealias ManagedObject = Intermediate.ManagedObject
+    private let batchSize: Int
+
+    public init(with persistentContainer: NSPersistentContainer, mergePolicyType: NSMergePolicyType = .mergeByPropertyObjectTrumpMergePolicyType, batchSize: Int = 1000) {
+        self.batchSize = batchSize
+        super.init(with: persistentContainer, mergePolicyType: mergePolicyType)
+    }
 
     open override func performWork(in context: NSManagedObjectContext) {
         do {
             let intermediates = try input.resolve()
+            let chunked = intermediates.chunked(into: batchSize)
 
-            ManagedObject.insertOrUpdate(intermediates: intermediates, in: context) { intermediate, managedObject in
-                intermediate.updateProperties(on: managedObject)
+            chunked.forEach { tasks in
+                ManagedObject.insertOrUpdate(intermediates: tasks, in: context) { intermediate, managedObject in
+                    intermediate.updateProperties(on: managedObject)
+                }
+                ManagedObject.insertOrUpdate(intermediates: tasks, in: context) { intermediate, managedObject in
+                    intermediate.updateRelationships(on: managedObject, in: context)
+                }
+
+                saveOperationContext()
             }
-
-            ManagedObject.insertOrUpdate(intermediates: intermediates, in: context) { intermediate, managedObject in
-                intermediate.updateRelationships(on: managedObject, in: context)
-            }
-
             saveAndFinish()
         } catch {
             output = Result { throw error }

PeakCoreDataTests/OperationTests.swift

@@ -182,6 +182,30 @@ class OperationTests: CoreDataTests, NSFetchedResultsControllerDelegate {
         waitForExpectations(timeout: defaultTimeout)
     }
 
+    func testChunkedBatchImportOperation() {
+        let numberOfItems = 1000
+
+        let finishExpectation = expectation(description: #function)
+
+        let input = CoreDataTests.createTestIntermediateObjects(number: numberOfItems, in: viewContext)
+        try! viewContext.save()
+
+        let operation = CoreDataBatchImportOperation<TestEntityJSON>(with: persistentContainer, batchSize: 200)
+        operation.input = Result { input }
+
+        operation.addResultBlock { result in
+            let outcome = try! result.resolve()
+            XCTAssertEqual(outcome.inserted.count, numberOfItems / 2)
+            XCTAssertEqual(outcome.updated.count, numberOfItems / 2)
+            XCTAssertEqual(outcome.all.count, numberOfItems)
+            finishExpectation.fulfill()
+        }
+
+        operationQueue.addOperation(operation)
+
+        waitForExpectations(timeout: defaultTimeout)
+    }
+
     func testComplexSaveOperation() {
         let finishExpectation = expectation(description: #function)
         let insertCount = 100
@@ -196,6 +220,45 @@ class OperationTests: CoreDataTests, NSFetchedResultsControllerDelegate {
         waitForExpectations(timeout: defaultTimeout)
     }
 
+
+    func testChunkedBatchImportPerformance_BatchSizeMax() {
+        let input = CoreDataTests.createTestIntermediateObjects(number: 10000, in: viewContext)
+        try! viewContext.save()
+
+        measure {
+            batchImport(items: input, batchSize: Int.max)
+        }
+    }
+
+    func testChunkedBatchImportPerformance_BatchSize1000() {
+        let input = CoreDataTests.createTestIntermediateObjects(number: 10000, in: viewContext)
+        try! viewContext.save()
+
+        measure {
+            batchImport(items: input, batchSize: 1000)
+        }
+    }
+
+    func testChunkedBatchImportPerformance_BatchSize200() {
+        let input = CoreDataTests.createTestIntermediateObjects(number: 10000, in: viewContext)
+        try! viewContext.save()
+
+        measure {
+            batchImport(items: input, batchSize: 200)
+        }
+    }
+
+    func batchImport(items: [TestEntityJSON], batchSize: Int) {
+        let operation = CoreDataBatchImportOperation<TestEntityJSON>(with: persistentContainer, batchSize: batchSize)
+        operation.input = Result { items }
+
+        let finishExpectation = expectation(description: #function)
+        operation.addResultBlock { result in
+            finishExpectation.fulfill()
+        }
+        operation.enqueue(on: operationQueue)
+        waitForExpectations(timeout: 60)
+    }
 }
 
 class FetchedResultsListener: NSObject, NSFetchedResultsControllerDelegate {

Podfile

@@ -1,8 +1,6 @@
 platform :ios, '10.0'
 
 target 'PeakCoreData' do
-  use_frameworks!
-
   pod 'PeakOperation'
 
   target 'PeakCoreDataTests' do

Podfile.lock

@@ -15,6 +15,6 @@ SPEC CHECKSUMS:
   PeakOperation: 1cfae299e91470e62e871bbab8d5e8d51820ea56
   PeakResult: a3bbeaae2485fec1199a09f6991a7b8613a9d6ed
 
-PODFILE CHECKSUM: bb53078b8e71ecb7337e38fb0df4e4291b5819a4
+PODFILE CHECKSUM: 0cf5a0ab55a1341aa394390bacf684acf27347ba
 
-COCOAPODS: 1.5.3
+COCOAPODS: 1.6.0