Fixed a major XCI compatibility bug by implementing compression/decom…

…pression support for NCA files with the first section having a smaller or larger offset then 0x4000. This fixes #49

nsz/BlockCompressor.py

@@ -34,7 +34,7 @@ def blockCompress(filePath, compressionLevel, blockSizeExponent, outputDir, thre
 
 def blockCompressContainer(readContainer, writeContainer, compressionLevel, blockSizeExponent, threads):
 	CHUNK_SZ = 0x100000
-	ncaHeaderSize = 0x4000
+	UNCOMPRESSABLE_HEADER_SIZE = 0x4000
 	if blockSizeExponent < 14 or blockSizeExponent > 32:
 		raise ValueError("Block size must be between 14 and 32")
 	blockSize = 2**blockSizeExponent
@@ -57,13 +57,15 @@ def blockCompressContainer(readContainer, writeContainer, compressionLevel, bloc
 		if isinstance(nspf, Nca.Nca) and nspf.header.contentType == Type.Content.DATA:
 			Print.info('Skipping delta fragment {0}'.format(nspf._path))
 			continue
-		if isinstance(nspf, Nca.Nca) and (nspf.header.contentType == Type.Content.PROGRAM or nspf.header.contentType == Type.Content.PUBLICDATA):
-			if isNcaPacked(nspf, ncaHeaderSize):
+		if isinstance(nspf, Nca.Nca) and (nspf.header.contentType == Type.Content.PROGRAM or nspf.header.contentType == Type.Content.PUBLICDATA) and nspf.size > UNCOMPRESSABLE_HEADER_SIZE:
+			if isNcaPacked(nspf):
+
+				offsetFirstSection = sortedFs(nspf)[0].offset
 				newFileName = nspf._path[0:-1] + 'z'
 				f = writeContainer.add(newFileName, nspf.size)
 				startPos = f.tell()
 				nspf.seek(0)
-				f.write(nspf.read(ncaHeaderSize))
+				f.write(nspf.read(UNCOMPRESSABLE_HEADER_SIZE))
 				sections = []
 
 				for fs in sortedFs(nspf):
@@ -92,7 +94,7 @@ def blockCompressContainer(readContainer, writeContainer, compressionLevel, bloc
 				chunkRelativeBlockID = 0
 				startChunkBlockID = 0
 				blocksHeaderFilePos = f.tell()
-				bytesToCompress = nspf.size - ncaHeaderSize
+				bytesToCompress = nspf.size - UNCOMPRESSABLE_HEADER_SIZE
 				blocksToCompress = bytesToCompress//blockSize + (bytesToCompress%blockSize > 0)
 				compressedblockSizeList = [0]*blocksToCompress
 				header = b'NCZBLOCK' #Magic
@@ -104,12 +106,21 @@ def blockCompressContainer(readContainer, writeContainer, compressionLevel, bloc
 				header += bytesToCompress.to_bytes(8, 'little') #Decompressed Size
 				header += b'\x00' * (blocksToCompress*4)
 				f.write(header)
-				decompressedBytes = ncaHeaderSize
+				decompressedBytes = UNCOMPRESSABLE_HEADER_SIZE
 				compressedBytes = f.tell()
 				BAR_FMT = u'{desc}{desc_pad}{percentage:3.0f}%|{bar}| {count:{len_total}d}/{total:d} {unit} [{elapsed}<{eta}, {rate:.2f}{unit_pad}{unit}/s]'
 				bar = enlighten.Counter(total=nspf.size//1048576, desc='Compressing', unit='MiB', color='cyan', bar_format=BAR_FMT)
 				subBars = bar.add_subcounter('green', all_fields=True)
-				partitions = [nspf.partition(offset = section.offset, size = section.size, n = None, cryptoType = section.cryptoType, cryptoKey = section.cryptoKey, cryptoCounter = bytearray(section.cryptoCounter), autoOpen = True) for section in sections]
+
+				partitions = []
+				if offsetFirstSection-UNCOMPRESSABLE_HEADER_SIZE > 0:
+					partitions.append(nspf.partition(offset = UNCOMPRESSABLE_HEADER_SIZE, size = offsetFirstSection-UNCOMPRESSABLE_HEADER_SIZE, cryptoType = Type.Crypto.CTR.NONE, autoOpen = True))
+				for section in sections:
+					#Print.info('offset: %x\t\tsize: %x\t\ttype: %d\t\tiv%s' % (section.offset, section.size, section.cryptoType, str(hx(section.cryptoCounter))), pleaseNoPrint)
+					partitions.append(nspf.partition(offset = section.offset, size = section.size, cryptoType = section.cryptoType, cryptoKey = section.cryptoKey, cryptoCounter = bytearray(section.cryptoCounter), autoOpen = True))
+				if UNCOMPRESSABLE_HEADER_SIZE-offsetFirstSection > 0:
+						partitions[0].seek(UNCOMPRESSABLE_HEADER_SIZE-offsetFirstSection)
+
 				partNr = 0
 				bar.count = nspf.tell()//1048576
 				subBars.count = f.tell()//1048576

nsz/Header.py

@@ -8,6 +8,12 @@ def __init__(self, f):
 		self.cryptoKey = f.read(16)
 		self.cryptoCounter = f.read(16)
 
+class FakeSection:
+	def __init__(self, offset, size):
+		self.offset = offset
+		self.size = size
+		self.cryptoType = 1
+
 class Block:
 	def __init__(self, f):
 		self.f = f

nsz/NszDecompressor.py

@@ -88,10 +88,10 @@ def __decompressContainer(readContainer, writeContainer, fileHashes, write, rais
 
 
 def __decompressNcz(nspf, f, statusReportInfo, pleaseNoPrint):
-	ncaHeaderSize = 0x4000
+	UNCOMPRESSABLE_HEADER_SIZE = 0x4000
 	blockID = 0
 	nspf.seek(0)
-	header = nspf.read(ncaHeaderSize)
+	header = nspf.read(UNCOMPRESSABLE_HEADER_SIZE)
 	if f != None:
 		start = f.tell()
 
@@ -100,7 +100,10 @@ def __decompressNcz(nspf, f, statusReportInfo, pleaseNoPrint):
 		raise ValueError("No NCZSECTN found! Is this really a .ncz file?")
 	sectionCount = nspf.readInt64()
 	sections = [Header.Section(nspf) for _ in range(sectionCount)]
-	nca_size = ncaHeaderSize
+	if sections[0].offset-UNCOMPRESSABLE_HEADER_SIZE > 0:
+		fakeSection = Header.FakeSection(UNCOMPRESSABLE_HEADER_SIZE, sections[0].offset-UNCOMPRESSABLE_HEADER_SIZE)
+		sections.insert(0, fakeSection)
+	nca_size = UNCOMPRESSABLE_HEADER_SIZE
 	for i in range(sectionCount):
 		nca_size += sections[i].size
 	pos = nspf.tell()
@@ -130,22 +133,30 @@ def __decompressNcz(nspf, f, statusReportInfo, pleaseNoPrint):
 		bar.refresh()
 	hash.update(header)
 
+	firstSection = True
 	for s in sections:
 		i = s.offset
-		crypto = aes128.AESCTR(s.cryptoKey, s.cryptoCounter)
+		useCrypto = s.cryptoType in (3, 4)
+		if useCrypto:
+			crypto = aes128.AESCTR(s.cryptoKey, s.cryptoCounter)
 		end = s.offset + s.size
+		if firstSection:
+			firstSection = False
+			uncompressedSize = UNCOMPRESSABLE_HEADER_SIZE-sections[0].offset
+			if uncompressedSize > 0:
+				i += uncompressedSize
 		while i < end:
-			crypto.seek(i)
+			if useCrypto:
+				crypto.seek(i)
 			chunkSz = 0x10000 if end - i > 0x10000 else end - i
 			if useBlockCompression:
 				inputChunk = blockDecompressorReader.read(chunkSz)
 			else:
 				inputChunk = decompressor.read(chunkSz)
+				decompressor.flush()
 			if not len(inputChunk):
 				break
-			if not useBlockCompression:
-				decompressor.flush()
-			if s.cryptoType in (3, 4):
+			if useCrypto:
 				inputChunk = crypto.encrypt(inputChunk)
 			if f != None:
 				f.write(inputChunk)

nsz/SectionFs.py

@@ -3,11 +3,11 @@ def sortedFs(nca):
 	fs.sort(key=lambda x: x.offset)
 	return fs
 
-def isNcaPacked(nca, ncaHeaderSize):
+def isNcaPacked(nca):
 	fs = sortedFs(nca)
 	if len(fs) == 0:
 		return True
-	next = ncaHeaderSize
+	next = fs[0].offset
 
 	for i in range(len(fs)):
 		if fs[i].offset != next:
@@ -16,4 +16,4 @@ def isNcaPacked(nca, ncaHeaderSize):
 
 	if next != nca.size:
 		return False
-	return True
+	return True

nsz/SolidCompressor.py

@@ -7,7 +7,7 @@
 from zstandard import FLUSH_FRAME, COMPRESSOBJ_FLUSH_FINISH, ZstdCompressor
 from PathTools import *
 
-ncaHeaderSize = 0x4000
+UNCOMPRESSABLE_HEADER_SIZE = 0x4000
 CHUNK_SZ = 0x1000000
 
 
@@ -23,16 +23,17 @@ def processContainer(readContainer, writeContainer, compressionLevel, threads, s
 			Print.info('Skipping delta fragment {0}'.format(nspf._path))
 			continue
 
-		if isinstance(nspf, Nca.Nca) and (nspf.header.contentType == Type.Content.PROGRAM or nspf.header.contentType == Type.Content.PUBLICDATA):
-			if isNcaPacked(nspf, ncaHeaderSize):
-
+		if isinstance(nspf, Nca.Nca) and (nspf.header.contentType == Type.Content.PROGRAM or nspf.header.contentType == Type.Content.PUBLICDATA) and nspf.size > UNCOMPRESSABLE_HEADER_SIZE:
+			if isNcaPacked(nspf):
+
+				offsetFirstSection = sortedFs(nspf)[0].offset
 				newFileName = nspf._path[0:-1] + 'z'
 
 				with writeContainer.add(newFileName, nspf.size, pleaseNoPrint) as f:
 					start = f.tell()
 
 					nspf.seek(0)
-					f.write(nspf.read(ncaHeaderSize))
+					f.write(nspf.read(UNCOMPRESSABLE_HEADER_SIZE))
 
 					sections = []
 					for fs in sortedFs(nspf):
@@ -62,17 +63,20 @@ def processContainer(readContainer, writeContainer, compressionLevel, threads, s
 					blocksHeaderFilePos = f.tell()
 					compressedblockSizeList = []
 
-					decompressedBytes = ncaHeaderSize
+					decompressedBytes = UNCOMPRESSABLE_HEADER_SIZE
 
 
 					stusReport[id] = [0, 0, nspf.size]
 
 					partitions = []
+					if offsetFirstSection-UNCOMPRESSABLE_HEADER_SIZE > 0:
+						partitions.append(nspf.partition(offset = UNCOMPRESSABLE_HEADER_SIZE, size = offsetFirstSection-UNCOMPRESSABLE_HEADER_SIZE, cryptoType = Type.Crypto.CTR.NONE, autoOpen = True))
 					for section in sections:
 						#Print.info('offset: %x\t\tsize: %x\t\ttype: %d\t\tiv%s' % (section.offset, section.size, section.cryptoType, str(hx(section.cryptoCounter))), pleaseNoPrint)
-						partitions.append(nspf.partition(offset = section.offset, size = section.size, n = None, cryptoType = section.cryptoType, cryptoKey = section.cryptoKey, cryptoCounter = bytearray(section.cryptoCounter), autoOpen = True))
-
-
+						partitions.append(nspf.partition(offset = section.offset, size = section.size, cryptoType = section.cryptoType, cryptoKey = section.cryptoKey, cryptoCounter = bytearray(section.cryptoCounter), autoOpen = True))
+					if UNCOMPRESSABLE_HEADER_SIZE-offsetFirstSection > 0:
+						partitions[0].seek(UNCOMPRESSABLE_HEADER_SIZE-offsetFirstSection)
+
 					partNr = 0
 					stusReport[id] = [nspf.tell(), f.tell(), nspf.size]
 					if threads > 1: