Merge branch 'develop'

.jazzy.yaml

@@ -3,11 +3,11 @@ sourcekitten_sourcefile: docs.json
 clean: false
 author: Timofey Solomko
 module: SWCompression
-module_version: 4.8.3
+module_version: 4.8.4
 copyright: '© 2022 Timofey Solomko'
 readme: README.md
 github_url: https://github.com/tsolomko/SWCompression
-github_file_prefix: https://github.com/tsolomko/SWCompression/tree/4.8.3
+github_file_prefix: https://github.com/tsolomko/SWCompression/tree/4.8.4
 theme: fullwidth
 
 custom_categories:

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## 4.8.4
+
+- Fixed an issue where in some cases BZip2 compression would produce incorrect output.
+- `TarReader` methods now always return `nil` after reaching the end of a TAR container.
+
 ## 4.8.3
 
 - There are now minimum deployment targets specified in Swift Package Manager manifest.

SWCompression.podspec

@@ -1,7 +1,7 @@
 Pod::Spec.new do |s|
 
   s.name         = "SWCompression"
-  s.version      = "4.8.3"
+  s.version      = "4.8.4"
   s.summary      = "A framework with functions for working with compression, archives and containers."
 
   s.description  = "A framework with (de)compression algorithms and functions for processing various archives and containers."

SWCompression.xcodeproj/SWCompression.plist

@@ -15,9 +15,9 @@
 	<key>CFBundlePackageType</key>
 	<string>FMWK</string>
 	<key>CFBundleShortVersionString</key>
-	<string>4.8.3</string>
+	<string>4.8.4</string>
 	<key>CFBundleVersion</key>
-	<string>88</string>
+	<string>89</string>
 	<key>NSHumanReadableCopyright</key>
 	<string>Copyright © 2022 Timofey Solomko</string>
 </dict>

SWCompression.xcodeproj/TestSWCompression.plist

@@ -15,8 +15,8 @@
 	<key>CFBundlePackageType</key>
 	<string>BNDL</string>
 	<key>CFBundleShortVersionString</key>
-	<string>4.8.3</string>
+	<string>4.8.4</string>
 	<key>CFBundleVersion</key>
-	<string>88</string>
+	<string>89</string>
 </dict>
 </plist>

SWCompression.xcodeproj/project.pbxproj

@@ -1147,7 +1147,7 @@
 			isa = PBXProject;
 			attributes = {
 				LastSwiftUpdateCheck = 0920;
-				LastUpgradeCheck = 1400;
+				LastUpgradeCheck = 1420;
 				ORGANIZATIONNAME = "Timofey Solomko";
 				TargetAttributes = {
 					06BE1AC71DB410F100EE0F59 = {
@@ -1528,7 +1528,7 @@
 				CLANG_WARN_SUSPICIOUS_MOVE = YES;
 				CLANG_WARN_UNREACHABLE_CODE = YES;
 				CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
-				CURRENT_PROJECT_VERSION = 88;
+				CURRENT_PROJECT_VERSION = 89;
 				DEAD_CODE_STRIPPING = YES;
 				DEBUG_INFORMATION_FORMAT = dwarf;
 				EAGER_LINKING = YES;
@@ -1613,7 +1613,7 @@
 				CLANG_WARN_SUSPICIOUS_MOVE = YES;
 				CLANG_WARN_UNREACHABLE_CODE = YES;
 				CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
-				CURRENT_PROJECT_VERSION = 88;
+				CURRENT_PROJECT_VERSION = 89;
 				DEAD_CODE_STRIPPING = YES;
 				DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
 				EAGER_LINKING = YES;
@@ -1678,7 +1678,7 @@
 				APPLICATION_EXTENSION_API_ONLY = YES;
 				DEFINES_MODULE = YES;
 				DYLIB_COMPATIBILITY_VERSION = 1;
-				DYLIB_CURRENT_VERSION = 88;
+				DYLIB_CURRENT_VERSION = 89;
 				DYLIB_INSTALL_NAME_BASE = "@rpath";
 				INFOPLIST_FILE = SWCompression.xcodeproj/SWCompression.plist;
 				INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Frameworks";
@@ -1705,7 +1705,7 @@
 				APPLICATION_EXTENSION_API_ONLY = YES;
 				DEFINES_MODULE = YES;
 				DYLIB_COMPATIBILITY_VERSION = 1;
-				DYLIB_CURRENT_VERSION = 88;
+				DYLIB_CURRENT_VERSION = 89;
 				DYLIB_INSTALL_NAME_BASE = "@rpath";
 				INFOPLIST_FILE = SWCompression.xcodeproj/SWCompression.plist;
 				INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Frameworks";

SWCompression.xcodeproj/xcshareddata/xcschemes/SWCompression.xcscheme

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <Scheme
-   LastUpgradeVersion = "1400"
+   LastUpgradeVersion = "1420"
    version = "1.7">
    <BuildAction
       parallelizeBuildables = "YES"

Sources/BZip2/BurrowsWheeler.swift

@@ -12,6 +12,7 @@ enum BurrowsWheeler {
         let suffixArray = SuffixArray.make(from: doubleBytes, with: 256)
         var bwt = [Int]()
         bwt.reserveCapacity(bytes.count)
+        // Pointer is an index in the transformed array, `bwt`, where the EOF marker would have been if we had used it.
         var pointer = 0
         for i in 1..<suffixArray.count {
             if suffixArray[i] < bytes.count {
@@ -21,7 +22,7 @@ enum BurrowsWheeler {
                     bwt.append(bytes.last!)
                 }
             } else if suffixArray[i] == bytes.count {
-                pointer = (i - 1) / 2
+                pointer = bwt.count
             }
         }
         return (bwt, pointer)

Sources/GZip/GzipHeader+ExtraField.swift

@@ -17,7 +17,7 @@ extension GzipHeader {
         /// Binary content of the extra field.
         public var bytes: [UInt8]
 
-        /// Initializes and extra field with the specified extra field (subfield) ID bytes and its binary content.
+        /// Initializes an extra field with the specified extra field (subfield) ID bytes and its binary content.
         public init(_ si1: UInt8, _ si2: UInt8, _ bytes: [UInt8]) {
             self.si1 = si1
             self.si2 = si2

Sources/TAR/TarReader.swift

@@ -53,15 +53,12 @@ public struct TarReader {
     }
 
     /**
-     Processes the next TAR entry by reading it from the container and calling the provided closure on the result.
+     Processes the next TAR entry by reading it from the container and calling the provided closure on the result. If
+     the argument supplied to the closure is `nil` this indicates that the end of the input was reached.
 
      On Darwin platforms both the reading and the call to the closure are performed inside the `autoreleasepool` which
      allows to reduce the peak memory usage.
 
-     If the argument supplied to the closure is `nil` this indicates that the end of the input was reached. After that
-     the repeated `TarReader.process(_:)` or `TarReader.read()` calls will result in the `DataError.truncated` being
-     thrown.
-
      - Throws: `DataError.truncated` if the input is truncated. `TarError` is thrown in case of malformed input. Errors
      thrown by `FileHandle` operations are also propagated.
      */
@@ -81,8 +78,7 @@ public struct TarReader {
      - Throws: `DataError.truncated` if the input is truncated. `TarError` is thrown in case of malformed input. Errors
      thrown by `FileHandle` operations are also propagated.
 
-     - Returns: The next entry from the container or `nil` if the end of the input has been reached. After that the
-     repeated `TarReader.process(_:)` or `TarReader.read()` calls will result in the `DataError.truncated` being thrown.
+     - Returns: The next entry from the container or `nil` if the end of the input has been reached.
      */
     public mutating func read() throws -> TarEntry? {
         let headerData = try getData(size: 512)
@@ -94,6 +90,10 @@ public struct TarReader {
             if try getData(size: 512) == Data(count: 512) {
                 return nil
             } else {
+                // In this case we have a zero-filled block immediately followed by a non-zero-filled block which do not
+                // match the EOF marker signature. In practice, this indicates a malformed TAR container, since a
+                // zero-filled block is not a valid TAR header (and in fact the end result is an error being thrown in
+                // TarHeader initializer later down the line).
                 try set(offset: offset)
             }
         } else if headerData.count < 512 {
@@ -162,12 +162,18 @@ public struct TarReader {
 
     private func getData(size: Int) throws -> Data {
         assert(size >= 0, "TarReader.getData(size:): negative size.")
-        guard size > 0
-            else { return Data() }
         if #available(macOS 10.15.4, iOS 13.4, watchOS 6.2, tvOS 13.4, *) {
-            guard let chunkData = try handle.read(upToCount: size)
-                else { throw DataError.truncated }
-            return chunkData
+            // The documentation for FileHandle.read(upToCount:) is a bit misleading. This method does "return the data
+            // obtained by reading length bytes starting at the current file pointer" even if the requested amount is
+            // larger than the available data. What is not clear is when the method returns nil. Apparently, there are
+            // (at least) two cases when it happens:
+            //  - the file pointer is at the EOF regardless of the argument value,
+            //  - the argument is zero.
+            // It is also unclear what happens when the argument is negative (it seems that it reads everything until
+            // the EOF), but the assertion above takes care of this. In any case, instead of returning nil we return
+            // empty data since both of these situations logically seem equivalent for our purposes. This also allows us
+            // to eliminate additional guard-check for the size parameter.
+            return try handle.read(upToCount: size) ?? Data()
         } else {
             // Technically, this can throw NSException, but since it is ObjC exception we cannot handle it in Swift.
             return handle.readData(ofLength: size)

Sources/swcomp/main.swift

@@ -7,7 +7,7 @@ import Foundation
 import SWCompression
 import SwiftCLI
 
-let _SWC_VERSION = "4.8.3"
+let _SWC_VERSION = "4.8.4"
 
 let cli = CLI(name: "swcomp", version: _SWC_VERSION,
               description: """

Tests/BZip2CompressionTests.swift

@@ -84,4 +84,14 @@ class BZip2CompressionTests: XCTestCase {
         try answerTest("test9")
     }
 
+    func testBurrowsWheelerRoundtrip() throws {
+        // This test is inspired by the reported issue #38 that uncovered a mistake with a pointer variable in BWT.
+        // "1"s can be anything (except zero), but it must be the same byte value in all places.
+        // Two consecutive zeros in the middle seem to be crucial for some reason.
+        let testData = Data([0, 1, 0, 1, 0, 0, 1, 0, 1])
+        let compressedData = BZip2.compress(data: testData)
+        let redecompressedData = try BZip2.decompress(data: compressedData)
+        XCTAssertEqual(redecompressedData, testData)
+    }
+
 }

Tests/TarReaderTests.swift

@@ -44,7 +44,6 @@ class TarReaderTests: XCTestCase {
             }
         }
         XCTAssertEqual(entriesCount, 1)
-
         try testHandle.closeCompat()
     }
 
@@ -104,6 +103,7 @@ class TarReaderTests: XCTestCase {
                 }
             }
             XCTAssertEqual(entriesCount, 1)
+            XCTAssertNil(try reader.read())
             try testHandle.closeCompat()
         }
     }
@@ -125,6 +125,7 @@ class TarReaderTests: XCTestCase {
                 }
             }
             XCTAssertEqual(entriesCount, 6)
+            XCTAssertNil(try reader.read())
             try testHandle.closeCompat()
         }
     }
@@ -158,6 +159,7 @@ class TarReaderTests: XCTestCase {
             XCTAssertNil(entry!.info.comment)
             XCTAssertEqual(entry!.data, "Hello, Windows!".data(using: .utf8))
         }
+        XCTAssertNil(try reader.read())
         try testHandle.closeCompat()
     }
 
@@ -178,6 +180,7 @@ class TarReaderTests: XCTestCase {
             XCTAssertEqual(entry!.data, Data())
 
         }
+        XCTAssertNil(try reader.read())
         try testHandle.closeCompat()
     }
 
@@ -197,6 +200,7 @@ class TarReaderTests: XCTestCase {
             XCTAssertNil(entry!.info.comment)
             XCTAssertNil(entry!.data)
         }
+        XCTAssertNil(try reader.read())
         try testHandle.closeCompat()
     }
 
@@ -217,8 +221,8 @@ class TarReaderTests: XCTestCase {
             XCTAssertEqual(entry!.info.permissions, Permissions(rawValue: 493))
             XCTAssertNil(entry!.info.comment)
             XCTAssertNil(entry!.data)
-
         }
+        XCTAssertNil(try reader.read())
         try testHandle.closeCompat()
     }
 
@@ -253,6 +257,7 @@ class TarReaderTests: XCTestCase {
             XCTAssertEqual(entry!.data, answerData)
 
         }
+        XCTAssertNil(try reader.read())
         try testHandle.closeCompat()
     }
 
@@ -273,6 +278,7 @@ class TarReaderTests: XCTestCase {
             XCTAssertEqual(entry!.data, answerData)
 
         }
+        XCTAssertNil(try reader.read())
         try testHandle.closeCompat()
     }
 
@@ -297,6 +303,7 @@ class TarReaderTests: XCTestCase {
             }
         }
         XCTAssertEqual(entriesCount, 3)
+        XCTAssertNil(try reader.read())
         try testHandle.closeCompat()
     }
 

azure-pipelines.yml

@@ -46,7 +46,7 @@ stages:
           UTILS_PW_XCF_FLAG: '--xcf'
         macosSwift57:
           imageName: 'macOS-12'
-          DEVELOPER_DIR: '/Applications/Xcode_14.0.1.app'
+          DEVELOPER_DIR: '/Applications/Xcode_14.2.app'
           WATCHOS_ACTIONS: 'clean test'
           WATCHOS_SIMULATOR: 'Apple Watch Series 6 (44mm)'
           UTILS_PW_XCF_FLAG: '--xcf'
@@ -108,7 +108,7 @@ stages:
           containerImage: 'swift:5.6.3-focal'
         linuxSwift57:
           imageName: 'ubuntu-20.04'
-          containerImage: 'swift:5.7-focal'
+          containerImage: 'swift:5.7.2-focal'
     pool:
       vmImage: $(imageName)
     container: $[ variables['containerImage'] ]
@@ -139,7 +139,7 @@ stages:
           SWIFT_DEV_PATH: 'C:\Library\Swift-development\bin'
         windowsSwift57:
           imageName: 'windows-2019'
-          SWIFT_VERSION: '5.7'
+          SWIFT_VERSION: '5.7.2'
           ICU_PATH: 'C:\Program Files\swift\icu-69.1\usr\bin'
           SWIFT_DEV_PATH: 'C:\Program Files\swift\runtime-development\usr\bin'
     pool:
@@ -174,7 +174,7 @@ stages:
     pool:
       vmImage: 'macOS-12'
     variables:
-      DEVELOPER_DIR: '/Applications/Xcode_14.0.1.app'
+      DEVELOPER_DIR: '/Applications/Xcode_14.2.app'
     steps:
     - script: |
         set -e -o xtrace