Issue #117: respect original check flags for ramdisk compression

ramdisk xz check flags: crc32/crc64 etc. XiaoMi recovery image uses CRC32 and can not suppport CRC64
cfig · May 29, 2023 · e7ed8fb · e7ed8fb
1 parent d0dfebd
commit e7ed8fb
Show file tree

Hide file tree

Showing 9 changed files with 149 additions and 23 deletions.
diff --git a/bbootimg/src/main/kotlin/bootimg/Common.kt b/bbootimg/src/main/kotlin/bootimg/Common.kt
@@ -258,7 +258,7 @@ class Common {
         }
 
         //using preset fs_config
-        fun packRootfs(rootDir: String, ramdiskGz: String) {
+        fun packRootfs(rootDir: String, ramdiskGz: String, compressorArgs: String? = null) {
             val root = File(rootDir).path
             log.info("Packing rootfs $root ...")
             when {
@@ -280,7 +280,7 @@ class Common {
                 ramdiskGz.endsWith(".xz") -> {
                     val f = ramdiskGz.removeSuffix(".xz")
                     AndroidCpio().pack(root, f, "${f}_filelist.txt")
-                    FileInputStream(f).use { ZipHelper.xz(ramdiskGz, it) }
+                    FileInputStream(f).use { ZipHelper.xz(ramdiskGz, it, compressorArgs!!) }
                 }
                 ramdiskGz.endsWith(".cpio") -> {
                     val f = ramdiskGz.removeSuffix(".cpio")

diff --git a/bbootimg/src/main/kotlin/bootimg/v2/BootV2.kt b/bbootimg/src/main/kotlin/bootimg/v2/BootV2.kt
@@ -16,11 +16,13 @@ package cfig.bootimg.v2
 
 import avb.AVBInfo
 import cfig.Avb
+import cfig.bootimg.Common as C
 import cfig.bootimg.Common
 import cfig.bootimg.Common.Companion.deleleIfExists
 import cfig.bootimg.Signer
 import cfig.helper.Dumpling
 import cfig.helper.Helper
+import cfig.helper.ZipHelper
 import cfig.packable.VBMetaParser
 import cfig.utils.DTC
 import cfig.utils.EnvironmentVerifier
@@ -39,7 +41,7 @@ import java.nio.ByteOrder
 data class BootV2(
     var info: MiscInfo = MiscInfo(),
     var kernel: CommArgs = CommArgs(),
-    var ramdisk: CommArgs = CommArgs(),
+    var ramdisk: RamdiskArgs = RamdiskArgs(),
     var secondBootloader: CommArgs? = null,
     var recoveryDtbo: CommArgsLong? = null,
     var dtb: CommArgsLong? = null,
@@ -68,6 +70,14 @@ data class BootV2(
         var loadOffset: Long = 0,
     )
 
+    data class RamdiskArgs(
+        var file: String? = null,
+        var position: Long = 0,
+        var size: Int = 0,
+        var loadOffset: Long = 0,
+        var xzFlags: String? = null
+    )
+
     data class CommArgsLong(
         var file: String? = null,
         var position: Long = 0,
@@ -193,10 +203,14 @@ data class BootV2(
         Common.dumpKernel(Helper.Slice(info.output, kernel.position.toInt(), kernel.size, kernel.file!!))
         //ramdisk
         if (this.ramdisk.size > 0) {
-            val fmt = Common.dumpRamdisk(
+            val fmt = C.dumpRamdisk(
                 Helper.Slice(info.output, ramdisk.position.toInt(), ramdisk.size, ramdisk.file!!), "${workDir}root"
             )
             this.ramdisk.file = this.ramdisk.file!! + ".$fmt"
+            if (fmt == "xz") {
+                val checkType = ZipHelper.xzStreamFlagCheckTypeToString(ZipHelper.parseStreamFlagCheckType(this.ramdisk.file!!))
+                this.ramdisk.xzFlags = checkType
+            }
             //dump info again
             mapper.writerWithDefaultPrettyPrinter().writeValue(File(workDir + this.info.json), this)
         }
@@ -398,7 +412,7 @@ data class BootV2(
                 File(this.ramdisk.file!!).deleleIfExists()
                 File(this.ramdisk.file!!.removeSuffix(".gz")).deleleIfExists()
                 //Common.packRootfs("${workDir}/root", this.ramdisk.file!!, Common.parseOsMajor(info.osVersion.toString()))
-                Common.packRootfs("${workDir}/root", this.ramdisk.file!!)
+                Common.packRootfs("${workDir}/root", this.ramdisk.file!!, this.ramdisk.xzFlags)
             }
             this.ramdisk.size = File(this.ramdisk.file!!).length().toInt()
         }

diff --git a/bbootimg/src/main/kotlin/bootimg/v2/BootV2Dialects.kt b/bbootimg/src/main/kotlin/bootimg/v2/BootV2Dialects.kt
@@ -37,7 +37,7 @@ import java.nio.ByteOrder
 data class BootV2Dialects(
     var info: MiscInfo = MiscInfo(),
     var kernel: CommArgs = CommArgs(),
-    var ramdisk: CommArgs = CommArgs(),
+    var ramdisk: BootV2.RamdiskArgs = BootV2.RamdiskArgs(),
     var secondBootloader: CommArgs? = null,
     var recoveryDtbo: CommArgsLong? = null,
     var dtb: CommArgsLong? = null,
@@ -378,7 +378,7 @@ data class BootV2Dialects(
                 File(this.ramdisk.file!!).deleleIfExists()
                 File(this.ramdisk.file!!.removeSuffix(".gz")).deleleIfExists()
                 //Common.packRootfs("${workDir}/root", this.ramdisk.file!!, Common.parseOsMajor(info.osVersion.toString()))
-                Common.packRootfs("${workDir}/root", this.ramdisk.file!!)
+                Common.packRootfs("${workDir}/root", this.ramdisk.file!!, this.ramdisk.xzFlags)
             }
             this.ramdisk.size = File(this.ramdisk.file!!).length().toInt()
         }

diff --git a/bbootimg/src/main/kotlin/bootimg/v3/BootV3.kt b/bbootimg/src/main/kotlin/bootimg/v3/BootV3.kt
@@ -25,6 +25,7 @@ import cfig.bootimg.Common.Companion.getPaddingSize
 import cfig.bootimg.Signer
 import cfig.helper.Helper
 import cfig.helper.Dumpling
+import cfig.helper.ZipHelper
 import cfig.packable.VBMetaParser
 import com.fasterxml.jackson.databind.ObjectMapper
 import de.vandermeer.asciitable.AsciiTable
@@ -41,7 +42,7 @@ import cfig.bootimg.Common as C
 data class BootV3(
     var info: MiscInfo = MiscInfo(),
     var kernel: CommArgs = CommArgs(),
-    val ramdisk: CommArgs = CommArgs(),
+    val ramdisk: RamdiskArgs = RamdiskArgs(),
     var bootSignature: CommArgs = CommArgs(),
 ) {
     companion object {
@@ -105,6 +106,13 @@ data class BootV3(
         var size: Int = 0,
     )
 
+    data class RamdiskArgs (
+        var file: String = "",
+        var position: Int = 0,
+        var size: Int = 0,
+        var xzFlags: String? = null
+    )
+
     fun pack(): BootV3 {
         if (this.kernel.size > 0) {
             this.kernel.size = File(this.kernel.file).length().toInt()
@@ -119,7 +127,7 @@ data class BootV3(
                 //TODO: remove cpio in C/C++
                 //C.packRootfs("$workDir/root", this.ramdisk.file, C.parseOsMajor(info.osVersion))
                 // enable advance JAVA cpio
-                C.packRootfs("$workDir/root", this.ramdisk.file)
+                C.packRootfs("$workDir/root", this.ramdisk.file, this.ramdisk.xzFlags)
             }
             this.ramdisk.size = File(this.ramdisk.file).length().toInt()
         }
@@ -232,6 +240,10 @@ data class BootV3(
                 Helper.Slice(info.output, ramdisk.position, ramdisk.size, ramdisk.file), "${workDir}root"
             )
             this.ramdisk.file = this.ramdisk.file + ".$fmt"
+            if (fmt == "xz") {
+                val checkType = ZipHelper.xzStreamFlagCheckTypeToString(ZipHelper.parseStreamFlagCheckType(this.ramdisk.file))
+                this.ramdisk.xzFlags = checkType
+            }
         }
         //bootsig
 

diff --git a/bbootimg/src/main/kotlin/bootimg/v3/VendorBoot.kt b/bbootimg/src/main/kotlin/bootimg/v3/VendorBoot.kt
@@ -22,6 +22,7 @@ import cfig.bootimg.Common.Companion.deleleIfExists
 import cfig.bootimg.Signer
 import cfig.helper.Dumpling
 import cfig.helper.Helper
+import cfig.helper.ZipHelper
 import cfig.packable.VBMetaParser
 import cfig.utils.DTC
 import cfig.utils.EnvironmentVerifier
@@ -37,7 +38,7 @@ import cfig.bootimg.Common as C
 
 data class VendorBoot(
     var info: MiscInfo = MiscInfo(),
-    var ramdisk: CommArgs = CommArgs(),
+    var ramdisk: RamdiskArgs = RamdiskArgs(),
     var dtb: CommArgs = CommArgs(),
     var ramdisk_table: Vrt = Vrt(),
     var bootconfig: CommArgs = CommArgs(),
@@ -49,6 +50,14 @@ data class VendorBoot(
         var loadAddr: Long = 0,
     )
 
+    data class RamdiskArgs(
+        var file: String = "",
+        var position: Long = 0,
+        var size: Int = 0,
+        var loadAddr: Long = 0,
+        var xzFlags: String? = null
+    )
+
     data class MiscInfo(
         var output: String = "",
         var json: String = "",
@@ -227,15 +236,15 @@ data class VendorBoot(
                     //Fixed: remove cpio in C/C++
                     //C.packRootfs("$workDir/root", this.ramdisk.file, parseOsMajor())
                     //enable advance JAVA cpio
-                    C.packRootfs("$workDir/root", this.ramdisk.file)
+                    C.packRootfs("$workDir/root", this.ramdisk.file, this.ramdisk.xzFlags)
                 }
                 this.ramdisk.size = File(this.ramdisk.file).length().toInt()
             }
             else -> {
                 this.ramdisk_table.ramdidks.forEachIndexed { index, it ->
                     File(it.file).deleleIfExists()
                     log.info(workDir + "root.${index + 1} -> " + it.file)
-                    C.packRootfs(workDir + "root.${index + 1}", it.file)
+                    C.packRootfs(workDir + "root.${index + 1}", it.file, this.ramdisk.xzFlags)
                 }
                 this.ramdisk.size = this.ramdisk_table.ramdidks.sumOf { File(it.file).length() }.toInt()
             }
@@ -345,6 +354,10 @@ data class VendorBoot(
             this.ramdisk_table.ramdidks.isEmpty())
         //@formatter:on
         this.ramdisk.file = this.ramdisk.file + ".$fmt"
+        if (fmt == "xz") {
+            val checkType = ZipHelper.xzStreamFlagCheckTypeToString(ZipHelper.parseStreamFlagCheckType(this.ramdisk.file))
+            this.ramdisk.xzFlags = checkType
+        }
         //dtb
         C.dumpDtb(Helper.Slice(info.output, dtb.position.toInt(), dtb.size, dtb.file))
         //vrt

diff --git a/bbootimg/src/main/kotlin/ota/DeltaGenerator.kt b/bbootimg/src/main/kotlin/ota/DeltaGenerator.kt
@@ -69,7 +69,7 @@ class DeltaGenerator {
                 //try xz
                 File.createTempFile("pre", "suf").let { tempFile ->
                     tempFile.deleteOnExit()
-                    ZipHelper.xz(tempFile.absolutePath, ByteArrayInputStream(inData))
+                    ZipHelper.xz(tempFile.absolutePath, ByteArrayInputStream(inData), "CRC64")
                     log.debug("raw=${inData.size}, xz=" + tempFile.length())
                     if (bestSize > tempFile.length()) {
                         bestType = Type.REPLACE_XZ

diff --git a/helper/src/main/kotlin/cfig/helper/ZipHelper.kt b/helper/src/main/kotlin/cfig/helper/ZipHelper.kt
@@ -17,11 +17,8 @@ package cfig.helper
 import cc.cfig.io.Struct
 import cfig.helper.Helper.Companion.check_call
 import cfig.helper.Helper.Companion.check_output
-import org.apache.commons.compress.archivers.zip.ZipArchiveEntry
-import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream
-import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream
-import org.apache.commons.compress.archivers.zip.ZipFile
-import org.apache.commons.compress.archivers.zip.ZipMethod
+import org.apache.commons.compress.archivers.zip.*
+import org.apache.commons.compress.compressors.CompressorOutputStream
 import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream
 import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream
 import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream
@@ -34,9 +31,10 @@ import org.apache.commons.exec.CommandLine
 import org.apache.commons.exec.DefaultExecutor
 import org.apache.commons.exec.PumpStreamHandler
 import org.slf4j.LoggerFactory
+import org.tukaani.xz.LZMA2Options
 import org.tukaani.xz.XZFormatException
+import org.tukaani.xz.XZOutputStream
 import java.io.*
-import java.lang.RuntimeException
 import java.net.URI
 import java.nio.file.FileSystems
 import java.nio.file.Files
@@ -50,6 +48,43 @@ import kotlin.reflect.jvm.isAccessible
 class ZipHelper {
     class ZipEntryRecipe(val data: ByteArray, val name: String, val method: ZipMethod)
 
+    class XZCompressorOutputStream2 : CompressorOutputStream {
+        private val out: XZOutputStream
+
+        constructor(outputStream: OutputStream?) {
+            out = XZOutputStream(outputStream, LZMA2Options())
+        }
+
+        constructor(outputStream: OutputStream?, checkType: Int) {
+            out = XZOutputStream(outputStream, LZMA2Options(), checkType)
+        }
+
+        @Throws(IOException::class)
+        override fun write(b: Int) {
+            out.write(b)
+        }
+
+        @Throws(IOException::class)
+        override fun write(buf: ByteArray, off: Int, len: Int) {
+            out.write(buf, off, len)
+        }
+
+        @Throws(IOException::class)
+        override fun flush() {
+            out.flush()
+        }
+
+        @Throws(IOException::class)
+        fun finish() {
+            out.finish()
+        }
+
+        @Throws(IOException::class)
+        override fun close() {
+            out.close()
+        }
+    }
+
     companion object {
         private val log = LoggerFactory.getLogger("ZipHelper")
 
@@ -86,9 +121,11 @@ class ZipHelper {
                             entryOut.mkdir()
                             log.debug("Unzipping[d]: ${entry.name}")
                         }
+
                         entry.isUnixSymlink -> {
                             throw IllegalArgumentException("this should not happen: Found dir ${entry.name}")
                         }
+
                         else -> {
                             val entryOut = File(outDir + "/" + entry.name)
                             log.debug("Unzipping[f]: ${entry.name}")
@@ -282,6 +319,54 @@ class ZipHelper {
             log.info("decompress(lzma) done: $compressedFile -> $decompressedFile")
         }
 
+        // https://tukaani.org/xz/xz-file-format.txt
+        // 2.1.1. Stream Header
+        fun parseStreamFlagCheckType(file: String): Int {
+            FileInputStream(file).use { fis ->
+                val ba = ByteArray(6)
+                check(fis.read(ba) == ba.size)
+                check(ba.contentEquals(byteArrayOf(0xfd.toByte(), 0x37, 0x7a, 0x58, 0x5a, 0x00))) {
+                    log.warn("wrong magic bytes in xz header")
+                }
+                check(fis.read(ba) == ba.size)
+                check(ba[0] == 0x00.toByte())
+                when (ba[1].toInt()) {
+                    0x00 -> log.info("NONE")
+                    0x01 -> log.info("CRC32")
+                    0x04 -> log.info("CRC64")
+                    0x0a -> log.info("SHA256")
+                    else -> throw IllegalArgumentException(
+                        "unsupported StreamFlag.CheckType: 0x" + ba[1].toInt().toString(16)
+                    )
+                }
+                return ba[1].toInt()
+            }
+        }
+
+        fun xzStreamFlagCheckTypeToString(type: Int): String {
+            return when (type) {
+                0x00 -> "NONE"
+                0x01 -> "CRC32"
+                0x04 -> "CRC64"
+                0x0a -> "SHA256"
+                else -> throw IllegalArgumentException(
+                    "unsupported StreamFlag.CheckType: 0x" + type.toString(16)
+                )
+            }
+        }
+
+        fun xzStreamFlagCheckTypeFromString(typeStr: String): Int {
+            return when (typeStr) {
+                "NONE" -> 0x00
+                "CRC32" -> 0x01
+                "CRC64" -> 0x04
+                "SHA256" -> 0x0a
+                else -> throw IllegalArgumentException(
+                    "unsupported StreamFlag.CheckType: 0x$typeStr"
+                )
+            }
+        }
+
         fun isXz(compressedFile: String): Boolean {
             return try {
                 FileInputStream(compressedFile).use { fis ->
@@ -294,10 +379,10 @@ class ZipHelper {
             }
         }
 
-        fun xz(compressedFile: String, fis: InputStream) {
-            log.info("Compress(xz) ... ")
+        fun xz(compressedFile: String, fis: InputStream, checkType: String) {
+            log.info("Compress(xz), with checkType $checkType... ")
             FileOutputStream(compressedFile).use { fos ->
-                XZCompressorOutputStream(fos).use { gos ->
+                XZCompressorOutputStream2(fos, ZipHelper.xzStreamFlagCheckTypeFromString(checkType)).use { gos ->
                     val buffer = ByteArray(1024)
                     while (true) {
                         val bytesRead = fis.read(buffer)

diff --git a/integrationTest.py b/integrationTest.py
@@ -176,6 +176,8 @@ def main():
     verifySingleDir(resDir2, "issue_91_unsigned_vendor_boot")
     # Issue 109: vendor_boot w/o dtb
     verifySingleDir(resDir2, "issue_109_vendor_boot_no_dtb")
+    # Issue 117: xz crc32/crc64
+    verifySingleDir(resDir2, "issue_117_xz_crc")
 
     log.info(successLogo)
 

diff --git a/src/integrationTest/resources_2 b/src/integrationTest/resources_2