Preserve file system compression when applying delta updates

Autoupdate/SPUDeltaArchive.m

@@ -81,16 +81,18 @@ @implementation SPUDeltaArchiveHeader
 
 @synthesize compression = _compression;
 @synthesize compressionLevel = _compressionLevel;
+@synthesize fileSystemCompression = _fileSystemCompression;
 @synthesize majorVersion = _majorVersion;
 @synthesize minorVersion = _minorVersion;
 
-- (instancetype)initWithCompression:(SPUDeltaCompressionMode)compression compressionLevel:(uint8_t)compressionLevel majorVersion:(uint16_t)majorVersion minorVersion:(uint16_t)minorVersion beforeTreeHash:(const unsigned char *)beforeTreeHash afterTreeHash:(const unsigned char *)afterTreeHash
+- (instancetype)initWithCompression:(SPUDeltaCompressionMode)compression compressionLevel:(uint8_t)compressionLevel fileSystemCompression:(bool)fileSystemCompression majorVersion:(uint16_t)majorVersion minorVersion:(uint16_t)minorVersion beforeTreeHash:(const unsigned char *)beforeTreeHash afterTreeHash:(const unsigned char *)afterTreeHash
 {
     self = [super init];
     if (self != nil)
     {
         _compression = compression;
         _compressionLevel = compressionLevel;
+        _fileSystemCompression = fileSystemCompression;
 
         _majorVersion = majorVersion;
         _minorVersion = minorVersion;

Autoupdate/SPUDeltaArchiveProtocol.h

@@ -26,10 +26,11 @@ typedef NS_ENUM(uint8_t, SPUDeltaItemCommands) {
 // Represents header for our archive
 @interface SPUDeltaArchiveHeader : NSObject
 
-- (instancetype)initWithCompression:(SPUDeltaCompressionMode)compression compressionLevel:(uint8_t)compressionLevel majorVersion:(uint16_t)majorVersion minorVersion:(uint16_t)minorVersion beforeTreeHash:(const unsigned char *)beforeTreeHash afterTreeHash:(const unsigned char *)afterTreeHash;
+- (instancetype)initWithCompression:(SPUDeltaCompressionMode)compression compressionLevel:(uint8_t)compressionLevel fileSystemCompression:(bool)fileSystemCompression majorVersion:(uint16_t)majorVersion minorVersion:(uint16_t)minorVersion beforeTreeHash:(const unsigned char *)beforeTreeHash afterTreeHash:(const unsigned char *)afterTreeHash;
 
 @property (nonatomic, readonly) SPUDeltaCompressionMode compression;
 @property (nonatomic, readonly) uint8_t compressionLevel;
+@property (nonatomic, readonly) bool fileSystemCompression;
 @property (nonatomic, readonly) uint16_t majorVersion;
 @property (nonatomic, readonly) uint16_t minorVersion;
 @property (nonatomic, readonly) unsigned char *beforeTreeHash;

Autoupdate/SPUSparkleDeltaArchive.h

@@ -39,7 +39,12 @@ NS_ASSUME_NONNULL_BEGIN
  [ HEADER (part 1) ]
  magic (length: 4)
  compression (length: 1)
- compressionLevel (length: 1)
+ metadata (See format below. length: 1)
+
+    -- METADATA --
+    compressionLevel (bits [0, 4])
+    (bits [5, 6] reserved)
+    fileSystemCompression (bit 7)
 
  -- COMPRESSED --
 

Autoupdate/SPUSparkleDeltaArchive.m

@@ -21,6 +21,13 @@
 #define SPARKLE_BZIP2_ERROR_DOMAIN @"Sparkle BZIP2"
 #define SPARKLE_COMPRESSION_ERROR_DOMAIN @"Sparkle Compression"
 
+typedef struct
+{
+    uint8_t compressionLevel : 4;
+    uint8_t reserved : 3;
+    bool fileSystemCompression : 1;
+} SparkleDeltaArchiveMetadata;
+
 @interface SPUSparkleDeltaArchive ()
 
 @property (nonatomic) FILE *file;
@@ -267,8 +274,8 @@ - (nullable SPUDeltaArchiveHeader *)readHeader
 
     self.compression = compression;
 
-    uint8_t compressionLevel = 0;
-    if (fread(&compressionLevel, sizeof(compressionLevel), 1, file) < 1) {
+    SparkleDeltaArchiveMetadata metadata = {0};
+    if (fread(&metadata, sizeof(metadata), 1, file) < 1) {
         self.error = [NSError errorWithDomain:NSPOSIXErrorDomain code:errno userInfo:@{ NSLocalizedDescriptionKey: [NSString stringWithFormat:@"Failed to read compression level value from patch file: %@", patchFile] }];
         return nil;
     }
@@ -338,7 +345,7 @@ - (nullable SPUDeltaArchiveHeader *)readHeader
         return nil;
     }
 
-    return [[SPUDeltaArchiveHeader alloc] initWithCompression:compression compressionLevel:compressionLevel majorVersion:majorVersion minorVersion:minorVersion beforeTreeHash:beforeTreeHash afterTreeHash:afterTreeHash];
+    return [[SPUDeltaArchiveHeader alloc] initWithCompression:compression compressionLevel:metadata.compressionLevel fileSystemCompression:metadata.fileSystemCompression majorVersion:majorVersion minorVersion:minorVersion beforeTreeHash:beforeTreeHash afterTreeHash:afterTreeHash];
 }
 
 - (NSArray<NSString *> *)_readRelativeFilePaths
@@ -770,8 +777,10 @@ - (void)writeHeader:(SPUDeltaArchiveHeader *)header
             compressionLevel = 0;
     }
 
-    if (fwrite(&compressionLevel, sizeof(compressionLevel), 1, file) < 1) {
-        self.error = [NSError errorWithDomain:NSPOSIXErrorDomain code:errno userInfo:@{ NSLocalizedDescriptionKey: @"Failed to write compression level value due to io error" }];
+    SparkleDeltaArchiveMetadata metadata = {.compressionLevel = compressionLevel, .fileSystemCompression = header.fileSystemCompression};
+
+    if (fwrite(&metadata, sizeof(metadata), 1, file) < 1) {
+        self.error = [NSError errorWithDomain:NSPOSIXErrorDomain code:errno userInfo:@{ NSLocalizedDescriptionKey: @"Failed to write metadata value due to io error" }];
         return;
     }
 

Autoupdate/SPUXarDeltaArchive.m

@@ -172,7 +172,7 @@ - (nullable SPUDeltaArchiveHeader *)readHeader
 
     // I wasn't able to figure out how to retrieve the compression options from xar,
     // so we will use default flags to indicate the info isn't available
-    return [[SPUDeltaArchiveHeader alloc] initWithCompression:SPUDeltaCompressionModeDefault compressionLevel:0 majorVersion:majorDiffVersion minorVersion:minorDiffVersion beforeTreeHash:rawExpectedBeforeHash afterTreeHash:rawExpectedAfterHash];
+    return [[SPUDeltaArchiveHeader alloc] initWithCompression:SPUDeltaCompressionModeDefault compressionLevel:0 fileSystemCompression:false majorVersion:majorDiffVersion minorVersion:minorDiffVersion beforeTreeHash:rawExpectedBeforeHash afterTreeHash:rawExpectedAfterHash];
 }
 
 - (void)writeHeader:(SPUDeltaArchiveHeader *)header

Autoupdate/SUBinaryDeltaApply.m

@@ -28,7 +28,7 @@ static BOOL applyBinaryDeltaToFile(NSString *patchFile, NSString *sourceFilePath
     return success;
 }
 
-BOOL applyBinaryDelta(NSString *source, NSString *destination, NSString *patchFile, BOOL verbose, void (^progressCallback)(double progress), NSError *__autoreleasing *error)
+BOOL applyBinaryDelta(NSString *source, NSString *finalDestination, NSString *patchFile, BOOL verbose, void (^progressCallback)(double progress), NSError *__autoreleasing *error)
 {
     SPUDeltaArchiveHeader *header = nil;
     id<SPUDeltaArchiveProtocol> archive = SPUDeltaArchiveReadPatchAndHeader(patchFile, &header);
@@ -39,7 +39,7 @@ BOOL applyBinaryDelta(NSString *source, NSString *destination, NSString *patchFi
         return NO;
     }
 
-    progressCallback(0/6.0);
+    progressCallback(0/7.0);
 
     SUBinaryDeltaMajorVersion majorDiffVersion = header.majorVersion;
     uint16_t minorDiffVersion = header.minorVersion;
@@ -90,7 +90,7 @@ BOOL applyBinaryDelta(NSString *source, NSString *destination, NSString *patchFi
         fprintf(stderr, "Verifying source...");
     }
 
-    progressCallback(1/6.0);
+    progressCallback(1/7.0);
 
     unsigned char beforeHash[CC_SHA1_DIGEST_LENGTH] = {0};
     if (!getRawHashOfTreeWithVersion(beforeHash, source, majorDiffVersion)) {
@@ -117,7 +117,17 @@ BOOL applyBinaryDelta(NSString *source, NSString *destination, NSString *patchFi
         fprintf(stderr, "\nCopying files...");
     }
 
-    progressCallback(2/6.0);
+    progressCallback(2/7.0);
+
+    // Make a temporary destination path if necessary
+    // If we want to apply file system compression after we're done applying, we'll need to use a different
+    // temporary path
+    NSString *destination;
+    if (header.fileSystemCompression) {
+        destination = [finalDestination.stringByDeletingLastPathComponent stringByAppendingPathComponent:[NSString stringWithFormat:@".tmp.%@", finalDestination.lastPathComponent]];
+    } else {
+        destination = finalDestination;
+    }
 
     if (!removeTree(destination)) {
         if (verbose) {
@@ -129,7 +139,7 @@ BOOL applyBinaryDelta(NSString *source, NSString *destination, NSString *patchFi
         return NO;
     }
 
-    progressCallback(3/6.0);
+    progressCallback(3/7.0);
 
     if (!copyTree(source, destination)) {
         if (verbose) {
@@ -141,7 +151,7 @@ BOOL applyBinaryDelta(NSString *source, NSString *destination, NSString *patchFi
         return NO;
     }
 
-    progressCallback(4/6.0);
+    progressCallback(4/7.0);
 
     if (verbose) {
         fprintf(stderr, "\nPatching...");
@@ -374,23 +384,85 @@ BOOL applyBinaryDelta(NSString *source, NSString *destination, NSString *patchFi
         if (error != NULL && *error == nil) {
             *error = archiveError;
         }
+        removeTree(destination);
         return NO;
     }
 
-    progressCallback(5/6.0);
-
+    progressCallback(5/7.0);
+
+    // Re-apply file system compression is requested
+    if (header.fileSystemCompression) {
+        if (verbose) {
+            fprintf(stderr, "\nApplying file system compression...");
+        }
+
+        NSTask *dittoTask = [[NSTask alloc] init];
+
+        if (@available(macOS 10.13, *)) {
+            dittoTask.executableURL = [NSURL fileURLWithPath:@"/usr/bin/ditto" isDirectory:NO];
+        } else {
+            dittoTask.launchPath = @"/usr/bin/ditto";
+        }
+
+        dittoTask.arguments = @[@"--hfsCompression", destination, finalDestination];
+
+        // If we fail to apply file system compression, we will try falling back to not doing this
+        BOOL failedToApplyFileSystemCompression = NO;
+
+        if (@available(macOS 10.13, *)) {
+            NSError *launchError = nil;
+            if (![dittoTask launchAndReturnError:&launchError]) {
+                failedToApplyFileSystemCompression = YES;
+
+                fprintf(stderr, "\nWarning: failed to launch ditto task for file compression: %s", launchError.localizedDescription.UTF8String);
+            }
+        } else {
+            @try {
+                [dittoTask launch];
+            } @catch (NSException *exception) {
+                failedToApplyFileSystemCompression = YES;
+
+                fprintf(stderr, "\nWarning: failed to launch ditto task for file compression: %s", exception.reason.UTF8String);
+            }
+        }
+
+        if (!failedToApplyFileSystemCompression) {
+            [dittoTask waitUntilExit];
+
+            if (dittoTask.terminationStatus != 0) {
+                failedToApplyFileSystemCompression = YES;
+
+                fprintf(stderr, "\nWarning: ditto task for file compression returned exit status %d", dittoTask.terminationStatus);
+            }
+        }
+
+        if (failedToApplyFileSystemCompression) {
+            // Try to replace bundle normally
+            if (![fileManager replaceItemAtURL:[NSURL fileURLWithPath:finalDestination] withItemAtURL:[NSURL fileURLWithPath:destination isDirectory:YES] backupItemName:nil options:0 resultingItemURL:NULL error:error]) {
+                removeTree(destination);
+                return NO;
+            }
+        } else {
+            // Remove original copy
+            [fileManager removeItemAtURL:[NSURL fileURLWithPath:destination isDirectory:YES] error:NULL];
+        }
+    }
+
+    progressCallback(6/7.0);
+
     if (verbose) {
         fprintf(stderr, "\nVerifying destination...");
     }
 
     unsigned char afterHash[CC_SHA1_DIGEST_LENGTH] = {0};
-    if (!getRawHashOfTreeWithVersion(afterHash, destination, majorDiffVersion)) {
+    if (!getRawHashOfTreeWithVersion(afterHash, finalDestination, majorDiffVersion)) {
         if (verbose) {
             fprintf(stderr, "\n");
         }
         if (error != NULL) {
-            *error = [NSError errorWithDomain:NSCocoaErrorDomain code:NSFileReadUnknownError userInfo:@{ NSLocalizedDescriptionKey: [NSString stringWithFormat:@"Unable to calculate hash of tree %@", destination] }];
+            *error = [NSError errorWithDomain:NSCocoaErrorDomain code:NSFileReadUnknownError userInfo:@{ NSLocalizedDescriptionKey: [NSString stringWithFormat:@"Unable to calculate hash of tree %@", finalDestination] }];
         }
+        removeTree(finalDestination);
         return NO;
     }
 
@@ -401,11 +473,11 @@ BOOL applyBinaryDelta(NSString *source, NSString *destination, NSString *patchFi
         if (error != NULL) {
             *error = [NSError errorWithDomain:NSCocoaErrorDomain code:NSFileReadUnknownError userInfo:@{ NSLocalizedDescriptionKey: [NSString stringWithFormat:@"Destination doesn't have expected hash (%@ != %@).  Giving up.", displayHashFromRawHash(expectedAfterHash), displayHashFromRawHash(afterHash)] }];
         }
-        removeTree(destination);
+        removeTree(finalDestination);
         return NO;
     }
 
-    progressCallback(6/6.0);
+    progressCallback(7/7.0);
 
     if (verbose) {
         fprintf(stderr, "\nDone!\n");

Autoupdate/SUBinaryDeltaCreate.m

@@ -517,6 +517,8 @@ BOOL createBinaryDelta(NSString *source, NSString *destination, NSString *patchF
         }
         return NO;
     }
+
+    bool foundFilesystemCompression = false;
 
     uint32_t warningsCount = 0;
     const uint32_t maxWarningsToPrint = 16;
@@ -584,6 +586,17 @@ BOOL createBinaryDelta(NSString *source, NSString *destination, NSString *patchF
                 fprintf(stderr, "\nWarning: encountered too many warnings.. Ignoring the rest..");
             }
         }
+
+        // If we find any executable files that are using file system compression, that is sufficient
+        // for recording that the applier should re-apply file system compression.
+        // We check for executable files because they are likely candidates to be compressed.
+        if (!foundFilesystemCompression && MAJOR_VERSION_IS_AT_LEAST(majorVersion, SUBinaryDeltaMajorVersion3) && ent->fts_info == FTS_F && (ent->fts_statp->st_mode & (S_IXUSR | S_IXGRP | S_IXOTH)) != 0 && (ent->fts_statp->st_flags & UF_COMPRESSED) != 0) {
+            foundFilesystemCompression = true;
+
+            if (verbose) {
+                fprintf(stderr, " File system compression detected.");
+            }
+        }
 
         NSDictionary *oldInfo = originalTreeState[key];
 
@@ -646,7 +659,7 @@ BOOL createBinaryDelta(NSString *source, NSString *destination, NSString *patchF
         archive = [[SPUXarDeltaArchive alloc] initWithPatchFileForWriting:temporaryFile];
     }
 
-    SPUDeltaArchiveHeader *header = [[SPUDeltaArchiveHeader alloc] initWithCompression:compression compressionLevel:compressionLevel majorVersion:majorVersion minorVersion:minorVersion beforeTreeHash:beforeHash afterTreeHash:afterHash];
+    SPUDeltaArchiveHeader *header = [[SPUDeltaArchiveHeader alloc] initWithCompression:compression compressionLevel:compressionLevel fileSystemCompression:foundFilesystemCompression majorVersion:majorVersion minorVersion:minorVersion beforeTreeHash:beforeHash afterTreeHash:afterHash];
 
     [archive writeHeader:header];
     if (archive.error != nil) {