Skip to content
Browse files

Initial import into git. Parts of the code are still a bit messy, but…

… it works very well.
  • Loading branch information...
0 parents commit 8498fd4ed90340df6fd100eaee6bf0406b64e88d @Plombo committed
Showing with 1,868 additions and 0 deletions.
  1. +167 −0 brrencode3.py
  2. +68 −0 ccfarchive.py
  3. +74 −0 huf8.py
  4. +142 −0 lz77.py
  5. +266 −0 lzh8.py
  6. +42 −0 nes_rom_extract.py
  7. +49 −0 romc.py
  8. +432 −0 romchu.py
  9. +135 −0 snesrestore.py
  10. +165 −0 u8archive.py
  11. +328 −0 wiimetadata.py
167 brrencode3.py
@@ -0,0 +1,167 @@
+#!/usr/bin/env python
+# Author: Bryan Cain
+# Based on but heavily modified from BRRTools by Bregalad (written in Java)
+# Date: January 16, 2011
+# Description: Encodes 16-bit signed PCM data to SNES BRR format.
+
+import wave
+import struct
+
+class BRREncoder(object):
+ def __init__(self, pcm, brr):
+ self.pcm_owner = False
+ self.brr_owner = False
+
+ if type(pcm) == type(''):
+ pcm = open(pcm, 'rb')
+ self.pcm_owner = True
+ if type(brr) == type(''):
+ brr = open(brr, 'wb')
+ self.brr_owner = True
+
+ self.pcm = pcm
+ self.brr = brr
+ self.p1 = 0
+ self.p2 = 0
+
+ # clamps value to a signed short
+ def sshort(self, n):
+ if n > 0x7FFF: return (n - 0x10000)
+ elif n < -0x8000: return n & 0x7FFF
+ else: return n
+
+ # short clamp_16(int n)
+ def clamp_16(self, n):
+ if n > 0x7FFF: return (0x7FFF - (n>>24))
+ else: return n
+
+ # void ADPCMBlockMash(short[] PCMData)
+ def ADPCMBlockMash(self, PCMData):
+ smin=0
+ kmin=0
+ dmin=2**31
+
+ for s in range(13, 0, -1):
+ for k in range(4):
+ d = self.ADPCMMash(s, k, PCMData, False)
+ if d < dmin:
+ kmin = k # Memorize the filter, shift values with smaller error
+ dmin = d
+ smin = s
+ if dmin == 0.0: break
+ if dmin == 0.0: break
+
+ self.BRRBuffer[0] = (smin<<4)|(kmin<<2)
+ self.ADPCMMash(smin, kmin, PCMData, True)
+
+ # double ADPCMMash(int shiftamount, int filter, short[] PCMData, boolean write)
+ def ADPCMMash(self, shiftamount, filter, PCMData, write):
+ d2=0.0
+ vlin=0
+ l1 = self.p1
+ l2 = self.p2
+ step = 1<<shiftamount
+
+ for i in range(16):
+ # Compute linear prediction for filters
+ if filter == 0:
+ pass
+ elif filter == 1:
+ vlin = l1 >> 1
+ vlin += (-l1) >> 5
+ elif filter == 2:
+ vlin = l1
+ vlin += (-(l1 +(l1>>1)))>>5
+ vlin -= l2 >> 1
+ vlin += l2 >> 5
+ else:
+ vlin = l1
+ vlin += (-(l1+(l1<<2) + (l1<<3)))>>7
+ vlin -= l2>>1
+ vlin += (l2+(l2>>1))>>4
+
+ d = (PCMData[i]>>1) - vlin # Difference between linear prediction and current sample
+ da = abs(d)
+
+ if da > 16384 and da < 32768:
+ d = d - 32768 * ( d >> 24 ) # Take advantage of wrapping
+ dp = d + (step << 2) + (step >> 2)
+ c = 0
+ if dp > 0:
+ if step > 1:
+ c = dp /(step>>1)
+ else:
+ c = dp<<1
+ if c > 15:
+ c = 15
+ c -= 8
+ dp = (c<<(shiftamount-1)) # quantized estimate of samp - vlin
+ # edge case, if caller even wants to use it */
+ if shiftamount > 12:
+ dp = ( dp >> 14 ) & ~0x7FF
+ c &= 0x0f # mask to 4 bits
+ l2 = l1 # shift history
+ l1 = self.sshort(self.clamp_16(vlin + dp)*2)
+ d = PCMData[i]-l1
+ d2 += float(d)*d # update square-error
+
+ if write: # if we want output, put it in proper place */
+ self.BRRBuffer[(i>>1)+1] |= c<<(4-((i&0x01)<<2))
+
+ if write:
+ self.p2 = l2
+ self.p1 = l1
+
+ return d2
+
+ # encodes the entire PCM file to BRR
+ def encode(self):
+ self.BRRBuffer = [0, 0, 0, 0, 0, 0, 0, 0, 0] # byte[9]
+
+ #wav = wave.open(wav, 'rb')
+ #if wav.getsampwidth() != 2: raise ValueError('must be 16 bits per sample')
+ pcm = self.pcm
+ brr = self.brr
+ self.p1 = 0
+ self.p2 = 0
+
+ samples2 = pcm.read(32) # the PCM samples in VC PCM files are misaligned for some reason
+ while len(samples2) == 32:
+ samples2 = struct.unpack('>16h', samples2)
+ self.BRRBuffer = [0, 0, 0, 0, 0, 0, 0, 0, 0]
+ self.ADPCMBlockMash(samples2)
+ brr.write(struct.pack('9B', *self.BRRBuffer))
+ #samples2 = wav.readframes(16)
+ samples2 = pcm.read(32)
+
+ #wav.close()
+ if self.pcm_owner: pcm.close()
+ if self.brr_owner: brr.close()
+
+ # offset: PCM offset (measured in samples, NOT in bytes)
+ # returns: 9-byte BRR block
+ def encode_block(self, offset):
+ # read PCM - the PCM samples in VC PCM files are misaligned for some reason
+ self.pcm.seek(offset * 2)
+ samples2 = self.pcm.read(32)
+
+ if len(samples2) != 32:
+ raise ValueError('invalid PCM offset %d (file offset %d)' % (offset, offset*2))
+
+ samples2 = struct.unpack('>16h', samples2)
+
+ # encode to BRR
+ self.BRRBuffer = [0, 0, 0, 0, 0, 0, 0, 0, 0]
+ self.ADPCMBlockMash(samples2)
+
+ return struct.pack('9B', *self.BRRBuffer)
+
+
+if __name__ == '__main__':
+ import sys
+ if len(sys.argv) != 3:
+ print 'Usage: %s input.pcm output.brr' % sys.argv[0]
+ enc = BRREncoder(sys.argv[1], sys.argv[2])
+ enc.encode()
+ print 'Wrote file %s' % sys.argv[2]
+
68 ccfarchive.py
@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+# Author: Bryan Cain (Plombo)
+# Date: December 27, 2010
+# Description: Reads Wii CCF archives, which contain Genesis and Master System ROMs.
+
+import struct
+import zlib
+from cStringIO import StringIO
+
+class CCFArchive(object):
+ # archive: a file-like object containing the CCF archive, OR the path to a CCF archive
+ def __init__(self, archive):
+ if type(archive) == type(''):
+ self.file = open(archive, 'rb')
+ else:
+ self.file = archive
+ self.files = []
+ self.readheader()
+
+ def readheader(self):
+ magic, zeroes1, rootnode_offset, numfiles, zeroes2 = struct.unpack('<4s12sII8s', self.file.read(32))
+ assert magic == 'CCF\0'
+ assert zeroes1 == 12 * '\0'
+ assert rootnode_offset == 0x20
+ assert zeroes2 == 8 * '\0'
+ for i in range(numfiles):
+ fd = FileDescriptor(self.file)
+ self.files.append(fd)
+
+ def hasfile(self, path):
+ for f in self.files:
+ if f.name == path: return True
+ return False
+
+ def getfile(self, path):
+ assert self.hasfile(path)
+ fd = None
+ for f in self.files:
+ if f.name == path: fd = f
+ return self.getfile2(fd)
+
+ def getfile2(self, fd):
+ self.file.seek(fd.data_offset * 32)
+ string = self.file.read(fd.size)
+ if fd.compressed:
+ string = zlib.decompress(string)
+ assert len(string) == fd.decompressed_size
+ return StringIO(string)
+
+ # returns the requested file, even if the name is cut off inside the archive
+ def find(self, name):
+ for fd in self.files:
+ if name.startswith(fd.name.rstrip()) or fd.name.startswith(name.rstrip()): return self.getfile2(fd)
+ return None
+
+class FileDescriptor(object):
+ # f: a file-like object of a CCF file at the position of this file descriptor
+ def __init__(self, f):
+ self.name, self.data_offset, self.size, self.decompressed_size = struct.unpack('<20sIII', f.read(32))
+ self.name = self.name[0:self.name.find('\0')]
+ self.compressed = (self.size != self.decompressed_size)
+
+if __name__ == '__main__':
+ import os
+ arc = CCFArchive(os.getenv('HOME') + '/wii/spinball/data.ccf')
+ arc.getfile('SonicSpinball_USA.S')
+
+
74 huf8.py
@@ -0,0 +1,74 @@
+#!/usr/bin/env python
+# Author: Bryan Cain (translated to Python from the "puff8" program by hcs, written in C)
+# Date: January 17, 2011
+# Description: Decompresses Nintendo's Huf8 compression used in Virtual Console games.
+
+import os, struct
+from array import array
+
+def decompress(infile, outfile):
+ infile.seek(0, os.SEEK_END)
+ file_length = infile.tell()
+ infile.seek(0, os.SEEK_SET)
+
+ # read header
+ magic_declength, symbol_count = struct.unpack('<IB', infile.read(5))
+ if (magic_declength & 0xFF) != 0x28:
+ raise ValueError("not 8-bit Huffman")
+ decoded_length = magic_declength >> 8
+ symbol_count += 1
+
+ # read decode table
+ decode_table_size = symbol_count * 2 - 1
+ decode_table = array('B', infile.read(decode_table_size))
+
+ '''
+ print "encoded size = %ld bytes (%d header + %ld body)" % (
+ file_length, 5 + decode_table_size,
+ file_length - (5 + decode_table_size))
+ print "decoded size = %ld bytes" % decoded_length
+ '''
+
+ # decode
+ bits = 0
+ bits_left = 0
+ table_offset = 0
+ bytes_decoded = 0
+
+ while bytes_decoded < decoded_length:
+ if bits_left == 0:
+ bits = struct.unpack("<I", infile.read(4))[0]
+ bits_left = 32
+
+ current_bit = ((bits & 0x80000000) != 0)
+ next_offset = (((table_offset + 1) / 2 * 2) + 1 +
+ (decode_table[table_offset] & 0x3f) * 2 +
+ current_bit)
+
+ if next_offset >= decode_table_size:
+ raise ValueError("reading past end of decode table")
+
+ if ((not current_bit and (decode_table[table_offset] & 0x80)) or
+ ( current_bit and (decode_table[table_offset] & 0x40))):
+ outfile.write(chr(decode_table[next_offset]))
+ bytes_decoded += 1
+ # print "%02x" % decode_table[next_offset]
+ next_offset = 0
+
+ if next_offset == table_offset:
+ raise ValueError("infinite loop in Huf8 decompression")
+ table_offset = next_offset
+ bits_left -= 1
+ bits <<= 1
+
+if __name__ == "__main__":
+ import sys
+ if len(sys.argv) != 3:
+ sys.stderr.write("Usage: %s infile outfile\n" % sys.argv[0])
+
+ infile = open(sys.argv[1], "rb")
+ outfile = open(sys.argv[2], "wb")
+
+ decompress(infile, outfile)
+ outfile.close()
+
142 lz77.py
@@ -0,0 +1,142 @@
+#!/usr/bin/env python
+# Author: Bryan Cain (Plombo)
+# Original WiiLZ77 class by Hector Martin (marcan)
+# Date: December 30, 2010
+# Description: Decompresses LZ77-encoded files and compressed N64 ROMs.
+
+import sys, os, struct
+from array import array
+from cStringIO import StringIO
+
+class BaseLZ77(object):
+ TYPE_LZ77_10 = 0x10
+ TYPE_LZ77_11 = 0x11
+
+ def uncompress(self):
+ if self.compression_type == self.TYPE_LZ77_11: return self.uncompress_11()
+ elif self.compression_type == self.TYPE_LZ77_10: return self.uncompress_10()
+ else: raise ValueError("Unsupported compression method %d"%self.compression_type)
+
+ def uncompress_10(self):
+ dout = array('c', '\0' * self.uncompressed_length)
+ offset = 0
+
+ self.file.seek(self.offset + 0x4)
+
+ while offset < self.uncompressed_length:
+ flags = ord(self.file.read(1))
+
+ for i in xrange(8):
+ if flags & 0x80:
+ info = struct.unpack(">H", self.file.read(2))[0]
+ num = 3 + (info>>12)
+ disp = info & 0xFFF
+ ptr = offset - disp - 1
+ for i in xrange(num):
+ dout[offset] = dout[ptr]
+ ptr += 1
+ offset += 1
+ if offset >= self.uncompressed_length:
+ break
+ else:
+ dout[offset] = self.file.read(1)
+ offset += 1
+ flags <<= 1
+ if offset >= self.uncompressed_length:
+ break
+
+ self.data = dout
+ return self.data
+
+ def uncompress_11(self):
+ dout = array('c', '\0'*self.uncompressed_length)
+ offset = 0
+
+ self.file.seek(self.offset + 0x4)
+
+ if not self.uncompressed_length:
+ self.uncompressed_length = struct.unpack("<I", self.file.read(4))[0]
+
+ while offset < self.uncompressed_length:
+ flags = ord(self.file.read(1))
+
+ for i in xrange(7, -1, -1):
+ if (flags & (1<<i)) > 0:
+ info = struct.unpack(">H", self.file.read(2))[0]
+ ptr, num = 0, 0
+ if info < 0x2000:
+ if info >= 0x1000:
+ info2 = struct.unpack(">H", self.file.read(2))[0]
+ ptr = offset - (info2 & 0xFFF) - 1
+ num = (((info & 0xFFF) << 4) | (info2 >> 12)) + 273
+ else:
+ info2 = ord(self.file.read(1))
+ ptr = offset - (((info & 0xF) << 8) | info2) - 1
+ num = ((info&0xFF0)>>4) + 17
+ else:
+ ptr = offset - (info & 0xFFF) - 1
+ num = (info>>12) + 1
+ for i in xrange(num):
+ dout[offset] = dout[ptr]
+ offset += 1
+ ptr += 1
+ if offset >= self.uncompressed_length:
+ break
+ else:
+ dout[offset] = self.file.read(1)
+ offset += 1
+
+ if offset >= self.uncompressed_length:
+ break
+
+ self.data = dout
+ return dout
+
+class WiiLZ77(BaseLZ77):
+ def __init__(self, file):
+ self.file = file
+ hdr = self.file.read(4)
+ if hdr != "LZ77":
+ self.file.seek(0)
+ self.offset = self.file.tell()
+
+ self.file.seek(0, os.SEEK_END)
+ self.compressed_length = self.file.tell()
+ self.file.seek(0, os.SEEK_SET)
+
+ hdr = struct.unpack("<I", self.file.read(4))[0]
+ self.uncompressed_length = hdr>>8
+ self.compression_type = hdr & 0xFF
+
+ #print "Compression type: 0x%02x" % self.compression_type
+ #print "Decompressed size: %d" % self.uncompressed_length
+
+def decompress(infile):
+ lz77obj = WiiLZ77(infile)
+ return StringIO(lz77obj.uncompress())
+
+def romc_decode(infile):
+ dec = RomcLZ77()
+ return dec.uncompress()
+
+if __name__ == '__main__':
+ import time
+ f = open(sys.argv[1])
+
+ start = time.clock()
+ unc = WiiLZ77(f)
+ try:
+ du = unc.uncompress_11()
+ except IndexError:
+ du = unc.data
+
+ end = time.clock()
+ print 'Time: %.2f seconds' % (end - start)
+
+ #du = romc_decode(f)
+
+ f2 = open(sys.argv[2],"w")
+ f2.write(''.join(du))
+ f2.close()
+ f.close()
+
266 lzh8.py
@@ -0,0 +1,266 @@
+#!/usr/bin/env python
+# Author: Bryan Cain
+# Original software (lzh8_dec 0.8) written in C by hcs
+# Date: January 17, 2011
+# Description: Decompresses Nintendo's N64 romc compression, type 2 (LZ77+Huffman)
+
+'''
+ LZH8 decompressor
+
+ An implementation of LZSS, with symbols stored via two Huffman codes:
+ - one for backreference lengths and literal bytes (8 bits each)
+ - one for backreference displacement lengths (bits - 1)
+
+ Layout of the compression:
+
+ 0x00: 0x40 (LZH8 identifier)
+ 0x01-0x03: uncompressed size (little endian)
+ 0x04-0x07: optional 32-bit size if 0x01-0x03 is 0
+ followed by:
+
+ 9-bit prefix coding tree table (for literal bytes and backreference lengths)
+ 0x00-0x01: Tree table size in 32-bit words, -1
+ 0x02-: Bit packed 9-bit inner nodes and leaves, stored as in Huff8
+ Total size: 2 ^ (leaf count + 1)
+
+ 5-bit prefix coding tree table (for backreference displacement lengths)
+ 0x00: Tree table size in 32-bit words, -1
+ 0x01-: Bit packed 5-bit inner nodes and leaves, stored as in Huff8
+ Total size: 2 ^ (leaf count + 1)
+
+ Followed by compressed data bitstream:
+ 1) Get a symbol from the 9-bit tree, if < 0x100 is a literal byte, repeat 1.
+ 2) If 1 wasn't a literal byte, symbol - 0x100 + 3 is the backreference length
+ 3) Get a symbol from the 5-bit tree, this is the length of the backreference
+ displacement.
+ 3a) If displacement length is zero, displacement is zero
+ 3b) If displacement length is one, displacement is one
+ 3c) If displacement length is > 1, displacement is next displen-1 bits,
+ with an extra 1 on the front (normalized).
+
+ Reverse engineered by hcs.
+'''
+
+import sys, os, struct
+from array import array
+
+VERSION = "0.8"
+
+# debug output options
+SHOW_SYMBOLS = 0
+SHOW_FREQUENCIES = 0
+SHOW_TREE = 0
+SHOW_TABLE = 0
+
+# constants
+LENBITS = 9
+DISPBITS = 5
+LENCNT = (1 << LENBITS)
+DISPCNT = (1 << DISPBITS)
+
+# globals
+input_offset = 0
+bit_pool = 0 # uint8_t
+bits_left = 0
+
+# read MSB->LSB order
+'''static inline uint16_t get_next_bits(
+ FILE *infile,
+ long * const offset_p,
+ uint8_t * const bit_pool_p,
+ int * const bits_left_p,
+ const int bit_count)'''
+def get_next_bits(infile, bit_count):
+ global input_offset, bit_pool, bits_left
+
+ offset_p = input_offset
+ bit_pool_p = bit_pool
+ bits_left_p = bits_left
+
+ out_bits = 0
+ num_bits_produced = 0
+ while num_bits_produced < bit_count:
+ if bits_left_p == 0:
+ infile.seek(offset_p)
+ bit_pool_p = struct.unpack("<B", infile.read(1))[0]
+ bits_left_p = 8
+ offset_p += 1
+
+ bits_this_round = 0
+ if bits_left_p > (bit_count - num_bits_produced):
+ bits_this_round = bit_count - num_bits_produced
+ else:
+ bits_this_round = bits_left_p
+
+ out_bits <<= bits_this_round
+ out_bits |= (bit_pool_p >> (bits_left_p - bits_this_round)) & ((1 << bits_this_round) - 1)
+
+ bits_left_p -= bits_this_round
+ num_bits_produced += bits_this_round
+
+ input_offset = offset_p
+ bit_pool = bit_pool_p
+ bits_left = bits_left_p
+
+ return out_bits
+
+# void analyze_LZH8(FILE *infile, FILE *outfile, long file_length)
+def decompress(infile):
+ global input_offset, bit_pool, bits_left
+ input_offset = 0
+ bit_pool = 0
+ bits_left = 0
+
+ # determine input file size
+ infile.seek(0, os.SEEK_END)
+ file_length = infile.tell()
+
+ # read header
+ infile.seek(input_offset)
+ header = struct.unpack("<I", infile.read(4))[0]
+ if (header & 0xFF) != 0x40: raise ValueError("not LZH8")
+ uncompressed_length = header >> 8
+ if uncompressed_length == 0:
+ uncompressed_length = struct.unpack("<I", f.read(4))[0]
+
+ # allocate output buffer
+ outbuf = array('B', '\0' * uncompressed_length) # uint8_t*
+
+ # allocate backreference length decode table
+ length_table_bytes = (struct.unpack("<H", infile.read(2))[0] + 1) * 4 # const uint32_t
+ length_decode_table_size = LENCNT * 2 # const long
+ length_decode_table = array('H', '\0' * length_decode_table_size * 2) # uint16_t* const
+
+ input_offset = infile.tell()
+
+ # read backreference length decode table
+ #if SHOW_TABLE: print "backreference length table"
+ start_input_offset = input_offset-2
+ i = 1
+ bits_left = 0
+ while (input_offset - start_input_offset) < length_table_bytes:
+ if i >= length_decode_table_size:
+ break
+ length_decode_table[i] = get_next_bits(infile, LENBITS)
+ i += 1
+ #if SHOW_TABLE: print "%ld: %d" % (i-1, length_decode_table[i-1])
+ input_offset = start_input_offset + length_table_bytes
+ bits_left = 0
+ #if SHOW_TABLE: print "done at 0x%lx" % input_offset
+
+ # allocate backreference displacement length decode table
+ infile.seek(input_offset)
+ displen_table_bytes = (struct.unpack("<B", infile.read(1))[0] + 1) * 4 # const uint32_t
+ input_offset += 1
+ displen_decode_table = array('B', '\0' * (DISPCNT * 2)) # uint8_t* const
+
+ # read backreference displacement length decode table
+ #if SHOW_TABLE: print "backreference displacement length table"
+ start_input_offset = input_offset-1
+ i = 1
+ bits_left = 0
+ while (input_offset - start_input_offset < displen_table_bytes):
+ if i >= length_decode_table_size:
+ break
+ displen_decode_table[i] = get_next_bits(infile, DISPBITS)
+ i += 1
+ #if SHOW_TABLE: print "%ld: %d" % (i-1, displen_decode_table[bit_pool = 0 # uint8_ti-1])
+ input_offset = start_input_offset + displen_table_bytes
+ bits_left = 0
+
+ #if SHOW_TABLE: print "done at 0x%lx" % input_offset
+
+ bytes_decoded = 0
+
+ # main decode loop
+ while bytes_decoded < uncompressed_length:
+ length_table_offset = 1
+
+ # get next backreference length or literal byte
+ while True:
+ next_length_child = get_next_bits(infile, 1)
+ length_node_payload = length_decode_table[length_table_offset] & 0x7F
+ next_length_table_offset = (length_table_offset / 2 * 2) + (length_node_payload + 1) * 2 + bool(next_length_child)
+ next_length_child_isleaf = length_decode_table[length_table_offset] & (0x100 >> next_length_child)
+
+ if next_length_child_isleaf:
+ length = length_decode_table[next_length_table_offset]
+
+ if 0x100 > length:
+ # literal byte
+ outbuf[bytes_decoded] = length
+ bytes_decoded += 1
+ else:
+ # backreference
+ length = (length & 0xFF) + 3
+ displen_table_offset = 1
+
+ # get backreference displacement length
+ while True:
+ next_displen_child = get_next_bits(infile, 1)
+ displen_node_payload = displen_decode_table[displen_table_offset] & 0x7
+ next_displen_table_offset = (displen_table_offset / 2 * 2) + (displen_node_payload + 1) * 2 + bool(next_displen_child)
+ next_displen_child_isleaf = displen_decode_table[displen_table_offset] & (0x10 >> next_displen_child)
+
+ if next_displen_child_isleaf:
+ displen = displen_decode_table[next_displen_table_offset]
+ displacement = 0
+
+ if displen != 0:
+ displacement = 1 # normalized
+
+ # collect the bits
+ #for (uint16_t i = displen-1; i > 0; i--)
+ for i in range(displen-1, 0, -1):
+ displacement *= 2
+ next_bit = get_next_bits(infile, 1)
+
+ displacement |= next_bit
+
+ # apply backreference
+ #for (long i = 0; i < length && bytes_decoded < uncompressed_length; bytes_decoded ++, i ++)
+ for i in range(length):
+ outbuf[bytes_decoded] = outbuf[bytes_decoded - displacement - 1]
+ bytes_decoded += 1
+ if bytes_decoded >= uncompressed_length: break
+
+ break # break out of displen tree traversal loop
+ else:
+ assert next_displen_table_offset != displen_table_offset # stuck in a loop somehow
+ displen_table_offset = next_displen_table_offset
+ # end of displen tree traversal loop
+ # end of if backreference !(0x100 > length)*/
+ break # break out of length tree traversal loop
+ else:
+ assert next_length_table_offset != length_table_offset # "stuck in a loop somehow"
+ length_table_offset = next_length_table_offset
+ # end of length tree traversal
+ # end of main decode loop
+
+ return outbuf.tostring()
+
+
+if __name__ == "__main__":
+ if len(sys.argv) != 3:
+ print "lzh8_dec %s\n" % VERSION
+ print "Usage: %s infile outfile" % sys.argv[0]
+ sys.exit(1)
+
+ # open file
+ infile = open(sys.argv[1], "rb")
+ outfile = open(sys.argv[2], "wb")
+
+ # decompress
+ print "Decompressing"
+ infile.seek(0, os.SEEK_SET)
+ output = decompress(infile)
+
+ print "Writing to file"
+ outfile.write(output)
+
+ outfile.close()
+ infile.close()
+
+ sys.exit(0)
+
+
42 nes_rom_extract.py
@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+# Author: Bryan Cain (Plombo)
+# Date: August 2010
+# Updated: December 28, 2010
+# Extracts an NES ROM from a 00000001.app file from an NES Virtual Console game.
+
+import sys
+from cStringIO import StringIO
+
+# returns a file-like object
+def extract_nes_rom(app1):
+ romoffset = 0
+ while True:
+ buf = app1.read(8192)
+ if buf.find('NES\x1a') >= 0: # Found NES ROM
+ romoffset += buf.find('NES\x1a')
+ break
+ elif len(buf) != 8192: # End of file, and no NES rom found
+ app1.close()
+ return None
+ else: romoffset += 8192
+
+ # NES ROM found; calculate size and extract it (FIXME: size calculation doesn't work)
+ app1.seek(romoffset)
+ #size = 16 + 128 # 16-byte header, 128-byte title data (footer)
+ #size += 16 * 1024 * ord(app1.read(1)) # next byte: number of PRG banks, 16KB each
+ #size += 8 * 1024 * ord(app1.read(1)) # next byte: number of CHR banks, 8KB each
+ app1.seek(romoffset)
+ rom = StringIO(app1.read())
+ return rom
+
+if __name__ == '__main__':
+ if len(sys.argv) != 3:
+ sys.exit('Usage: %s input.app output.nes' % sys.argv[0])
+ f = open(sys.argv[1], 'rb')
+ rom = extract_nes_rom(f)
+ f.close()
+ f2 = open(sys.argv[2], 'wb')
+ f2.write(rom.read())
+ f2.close()
+ print 'Done!'
+
49 romc.py
@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+# Author: Bryan Cain
+# Date: January 17, 2011
+# Description: Decompresses Nintendo's romc compression used in N64 VC games.
+
+import lz77, romchu, struct
+
+class RomcLZ77(lz77.BaseLZ77):
+ FOURMBYTE = 4194304 # 4MB rom size
+
+ def __init__(self, file):
+ self.file = file
+ self.offset = 0
+ self.uncompressed_length = self.FOURMBYTE * struct.unpack(">BBBB", self.file.read(4))[0]
+ self.compression_type = self.TYPE_LZ77_10
+
+def decompress(infile):
+ # read compression type
+ infile.seek(0)
+ compression_type = struct.unpack(">BBBB", infile.read(4))[3] & 0x3
+
+ # decompress
+ infile.seek(0)
+ if compression_type == 0x01: # LZ77/LZSS
+ dec = RomcLZ77(infile)
+ return dec.uncompress()
+ elif compression_type == 0x02: # LZ77+Huffman (romchu)
+ return romchu.decompress(infile)
+ else:
+ raise ValueError("unknown romc compression type %d" % compression_type)
+
+if __name__ == '__main__':
+ import sys, time
+
+ if len(sys.argv) != 3:
+ print 'Usage: %s infile outfile' % sys.argv[0]
+ sys.exit(1)
+
+ infile = open(sys.argv[1], 'rb')
+ start = time.clock()
+ output = decompress(infile)
+ end = time.clock()
+ print 'Time: %.2f seconds' % (end - start)
+
+ outfile = open(sys.argv[2], 'wb')
+ outfile.write(output)
+ outfile.close()
+ infile.close()
+
432 romchu.py
@@ -0,0 +1,432 @@
+#!/usr/bin/env python
+# Author: Bryan Cain
+# Original software (romchu 0.3) written in C by hcs
+# Date: January 17, 2011
+# Description: Decompresses Nintendo's N64 romc compression, type 2 (LZ77+Huffman)
+
+import sys, struct, time
+from array import array
+
+VERSION = "0.6"
+
+class backref(object):
+ def __init__(self):
+ self.bits = 0
+ self.base = 0
+
+#backref backref_len[0x1D], backref_disp[0x1E];
+backref_len, backref_disp = [], []
+for i in xrange(0x1D):
+ backref_len.append(backref())
+for i in xrange(0x1E):
+ backref_disp.append(backref())
+
+def main():
+ if len(sys.argv) != 3:
+ sys.stderr.write("romchu %s - romc type 2 decompressor\n" % VERSION)
+ sys.stderr.write("Usage: %s romc out.n64\n", sys.argv[0]);
+ sys.exit(1)
+
+ infile = open(sys.argv[1], "rb")
+ outfile = open(sys.argv[2], "wb")
+
+ outfile.write(decompress(infile))
+ outfile.close()
+ infile.close()
+ print "ok!"
+
+def decompress(infile):
+ block_mult = 0x10000
+ block_count = 0
+ out_offset = 0
+
+ # read header
+ infile.seek(0)
+ head_buf = infile.read(4)
+ #bs = init_bitstream(head_buf, 0, 4*8)
+
+ nominal_size = ord(head_buf[0])
+ nominal_size *= 0x100
+ nominal_size |= ord(head_buf[1])
+ nominal_size *= 0x100
+ nominal_size |= ord(head_buf[2])
+ nominal_size *= 0x40;
+ nominal_size |= ord(head_buf[3]) >> 2
+ romc_type = ord(head_buf[3]) & 0x3
+
+ if romc_type != 2:
+ raise ValueError("Expected type 2 romc, got %d\n" % romc_type)
+
+ #free_bitstream(bs)
+
+ # initialize backreference lookup tables
+ for i in xrange(8):
+ backref_len[i].bits = 0
+ backref_len[i].base = i
+
+ i = 8
+ for scale in xrange(1, 6):
+ k = (1<<(scale+2))
+ while k < (1<<(scale+3)):
+ backref_len[i].bits = scale;
+ backref_len[i].base = k;
+ k += (1<<scale)
+ i += 1
+
+ backref_len[28].bits = 0
+ backref_len[28].base = 255
+
+ for i in xrange(4):
+ backref_disp[i].bits = 0
+ backref_disp[i].base = i
+
+ i = 4
+ k = 4
+ #for (unsigned int i = 4, scale = 1, k = 4; scale < 14; scale ++)
+ for scale in xrange(1, 14):
+ #for (unsigned int j = 0; j < 2; j ++, k += (1 << scale), i++)
+ for j in xrange(2):
+ backref_disp[i].bits = scale
+ backref_disp[i].base = k
+ k += (1 << scale)
+ i += 1
+
+ # be lazy and just allocate memory for the whole file
+ out_buf = array('B', '\0' * nominal_size)
+ out_offset = 0
+
+ start = time.clock()
+
+ # decode each block
+ while True:
+ head_buf = infile.read(4)
+ if len(head_buf) != 4: break
+
+ '''
+ printf("%08lx=%08lx\n",
+ (unsigned long)(ftell(infile)-4),
+ (unsigned long)block_count*block_mult);
+ '''
+
+ head_bs = init_bitstream(head_buf, 0, 4*8)
+
+ compression_flag = get_bits(head_bs, 1)
+ if compression_flag: # compressed
+ # number of bits, including this header
+ block_size = get_bits(head_bs, 31) - 32
+ payload_bytes = block_size/8
+ payload_bits = block_size%8
+ else: # uncompressed
+ # number of bytes
+ block_size = get_bits(head_bs, 31)
+ payload_bytes = block_size
+ payload_bits = 0
+
+ #free_bitstream(head_bs)
+ head_bs = None
+
+ # read payload
+ read_size = payload_bytes
+ if payload_bits > 0:
+ read_size += 1
+
+ #this is not needed in Python
+ '''if read_size > len(payload_buf):
+ raise ValueError("payload too large")'''
+
+ payload_buf = infile.read(read_size)
+
+ # attempt to parse...
+ if compression_flag:
+ # read table 1 size
+ tab1_offset = 0
+ bs = init_bitstream(payload_buf, tab1_offset, payload_bytes*8+payload_bits)
+ tab1_size = get_bits(bs, 16)
+ #free_bitstream(bs)
+
+ # load table 1
+ bs = init_bitstream(payload_buf, tab1_offset + 2, tab1_size)
+ table1 = load_table(bs, 0x11D)
+ #free_bitstream(bs)
+
+ # read table 2 size
+ tab2_offset = tab1_offset + 2 + (tab1_size+7) / 8
+ bs = init_bitstream(payload_buf, tab2_offset, 2*8)
+ tab2_size = get_bits(bs, 16)
+ #free_bitstream(bs)
+
+ # load table 2
+ bs = init_bitstream(payload_buf, tab2_offset + 2, tab2_size)
+ table2 = load_table(bs, 0x1E)
+ #free_bitstream(bs)
+
+ # decode body
+ body_offset = tab2_offset + 2 + (tab2_size+7) / 8
+ body_size = payload_bytes*8 + payload_bits - body_offset*8
+ bs = init_bitstream(payload_buf, body_offset, body_size)
+
+ while not bitstream_eof(bs):
+ symbol = huf_lookup(bs, table1)
+
+ if symbol < 0x100:
+ # byte literal
+ #unsigned char b = symbol;
+ b = symbol
+ assert out_offset <= nominal_size # generated too many bytes
+ out_buf[out_offset] = b
+ out_offset += 1
+ else:
+ # backreference
+ len_bits = backref_len[symbol-0x100].bits
+ length = backref_len[symbol-0x100].base
+ if len_bits > 0:
+ length += get_bits(bs, len_bits)
+ length += 3
+
+ symbol2 = huf_lookup(bs, table2)
+
+ disp_bits = backref_disp[symbol2].bits
+ disp = backref_disp[symbol2].base
+ if disp_bits > 0:
+ disp += get_bits(bs, disp_bits)
+ disp += 1
+
+ assert disp <= out_offset # backreference too far
+ assert (out_offset + length) <= nominal_size # generated too many bytes
+
+ #for i in range(length):
+ count = 0
+ while count < length:
+ #for i in range(length):
+ out_buf[out_offset] = out_buf[out_offset-disp]
+ out_offset += 1
+ count += 1
+
+ #free_table(table1)
+ #free_table(table2)
+ #free_bitstream(bs)
+ else: # not compression_flag
+ assert (out_offset + payload_bytes) <= nominal_size # generated too many bytes
+ out_buf[out_offset:out_offset+payload_bytes] = payload_buf[0:payload_bytes]
+ out_offset += payload_bytes
+
+ block_count += 1
+ sys.stdout.write("\rDecompressed %d of %d bytes [%x/%x] (%5.2f%%)" % (out_offset, nominal_size, out_offset, nominal_size, 100.0 * out_offset / nominal_size))
+ sys.stdout.flush()
+ #print '\nDecompressed block %d in %.2f seconds' % (block_count, time.clock() - start)
+
+ print # start a new line after the progress counter
+ assert out_offset == nominal_size # size mismatch
+
+ #print 'Average block time: %.2f seconds' % (time.clock() / block_count)
+
+ return out_buf
+
+
+# bitstream reader
+class bitstream(object):
+ def __init__(self):
+ #const unsigned char* pool
+ #long bits_left
+ #uint8_t first_byte
+ #int first_byte_bits
+ self.pool = None
+ self.bits_left = 0
+ self.first_byte = 0
+ self.first_byte_bits = 0
+ self.index = 0
+
+# struct bitstream *init_bitstream(const unsigned char *pool, unsigned long pool_size)
+def init_bitstream(pool_buf, pool_start, pool_size):
+ bs = bitstream()
+
+ bs.pool = array('B', pool_buf[pool_start:pool_start+pool_size])
+ bs.bits_left = pool_size
+ bs.first_byte_bits = 0
+
+ # check that padding bits are 0 (to ensure we aren't ignoring anything)
+ if pool_size % 8:
+ if bs.pool[pool_size/8] & ~((1<<(pool_size%8))-1):
+ raise ValueError("nonzero padding at end of bitstream")
+
+ return bs
+
+# uint32_t get_bits(struct bitstream *bs, int bits)
+def get_bits(bs, bits):
+ accum = 0
+
+ if bits > 32:
+ raise ValueError("get_bits() supports max 32")
+ if bits > (bs.bits_left + bs.first_byte_bits):
+ raise ValueError("get_bits() underflow")
+
+ count = 0
+ #for i in range(bits):
+ while count < bits:
+ if bs.first_byte_bits == 0:
+ #print bs.bits_left, len(bs.pool), i
+ bs.first_byte = bs.pool[bs.index]
+ bs.index += 1
+ if bs.bits_left >= 8:
+ bs.first_byte_bits = 8
+ bs.bits_left -= 8
+ else:
+ bs.first_byte_bits = bs.bits_left
+ bs.bits_left = 0
+
+ accum >>= 1
+ accum |= (bs.first_byte & 1) << 31
+ bs.first_byte >>= 1
+ bs.first_byte_bits -= 1
+ count += 1
+
+ return accum >> (32-bits)
+
+def bitstream_eof(bs):
+ return (bs.bits_left + bs.first_byte_bits == 0)
+
+def free_bitstream(bs):
+ pass
+
+# Huffman code handling
+'''class hufnode_inner(object):
+ def __init__(self):
+ self.left, self.right = 0, 0
+
+class hufnode_leaf(object):
+ def __init__(self):
+ self.symbol = 0
+
+class hufnode_union(object):
+ def __init__(self):
+ self.inner = hufnode_inner()
+ self.leaf = hufnode_leaf()'''
+
+class hufnode(object):
+ def __init__(self):
+ self.is_leaf = False
+ self.symbol = 0
+ self.left = 0
+ self.right = 0
+ #self.u = hufnode_union()
+
+class huftable(object):
+ def __init__(self):
+ self.symbols = 0
+ self.t = []
+
+# struct huftable *load_table(struct bitstream *bs, int symbols)
+def load_table(bs, symbols):
+ len_count = [0] * 32
+ codes = [0] * 32
+ length_of = [0] * symbols
+ i = 0
+
+ while i < symbols:
+ if get_bits(bs, 1):
+ # run of equal lengths
+ count = get_bits(bs, 7) + 2
+ length = get_bits(bs, 5)
+
+ len_count[length] += count
+ for j in xrange(count):
+ length_of[i] = length
+ i += 1
+ else:
+ # set of inequal lengths
+ count = get_bits(bs, 7) + 1
+
+ for j in xrange(count):
+ length = get_bits(bs, 5)
+ length_of[i] = length
+ len_count[length] += 1
+ i += 1
+
+ assert bitstream_eof(bs) # did not exhaust bitstream reading table
+
+ # compute the first canonical Hufman code for each length
+ accum = 0
+ for i in xrange(1, 32):
+ accum = (accum + len_count[i-1]) << 1
+ codes[i] = accum
+
+ # determine codes and build a tree
+ ht = huftable()
+ ht.symbols = symbols
+ for i in xrange(symbols * 2):
+ node = hufnode()
+ node.is_leaf = 0
+ node.left = 0
+ node.right = 0
+ ht.t.append(node)
+
+ next_free_node = 1
+ for i in xrange(symbols):
+ cur = 0
+ if length_of[i] == 0:
+ # 0 length indicates absent symbol
+ continue
+
+ #for (int j = length_of[i]-1; j >= 0; j --)
+ for j in xrange(length_of[i]-1, -1, -1):
+ #next = 0 # shouldn't be necessary
+ assert not ht.t[cur].is_leaf # oops, walked onto a leaf
+
+ if codes[length_of[i]] & (1<<j):
+ # 1 == right
+ next = ht.t[cur].right
+ if 0 == next:
+ next = next_free_node
+ ht.t[cur].right = next
+ next_free_node += 1
+ else:
+ # 0 == left
+ next = ht.t[cur].left
+ if 0 == next:
+ next = next_free_node
+ ht.t[cur].left = next
+ next_free_node += 1
+
+ cur = next
+
+ ht.t[cur].is_leaf = 1
+ ht.t[cur].symbol = i
+
+ codes[length_of[i]] += 1
+
+ return ht
+
+# int huf_lookup(struct bitstream *bs, struct huftable *ht)
+def huf_lookup(bs, ht):
+ cur = 0
+ while not ht.t[cur].is_leaf:
+ if bs.first_byte_bits == 0:
+ bs.first_byte = bs.pool[bs.index]
+ bs.index += 1
+ if bs.bits_left >= 8:
+ bs.first_byte_bits = 8
+ bs.bits_left -= 8
+ else:
+ bs.first_byte_bits = bs.bits_left
+ bs.bits_left = 0
+
+ #if get_bits(bs, 1):
+ if bs.first_byte & 1:
+ # 1 == right
+ cur = ht.t[cur].right
+ else:
+ cur = ht.t[cur].left
+
+ bs.first_byte >>= 1
+ bs.first_byte_bits -= 1
+
+ return ht.t[cur].symbol
+
+# void free_table(struct huftable *ht)
+def free_table(ht):
+ pass
+
+
+if __name__ == "__main__":
+ main()
+
135 snesrestore.py
@@ -0,0 +1,135 @@
+#!/usr/bin/env python
+# Author: Bryan Cain
+# Date: December 31, 2010
+# Description: Attempts to restore the original sound samples to SNES ROMs in
+# Virtual Console games, which use uncompressed PCM data separate from the ROM.
+
+import os
+import sys
+import struct
+from brrencode3 import BRREncoder
+from cStringIO import StringIO
+
+# vcrom: file-like object for the original VC ROM
+# brr: file-like object containing the BRR-compressed audio samples
+# Precondition: brr is the correct size
+def restore_brr_samples(vcrom, pcm):
+ # read the samples from the input ROM into memory (TODO: check file size first)
+ vcrom.seek(0)
+ str = vcrom.read()
+ samplestart = str.find('PCMF') # samples start with first instance of the string "PCMF"
+ str = str[samplestart:]
+
+ # initialize output ROM in memory as a StringIO (pseudo-file)
+ rom = StringIO()
+ rom.write(str)
+
+ # read the input BRR samples
+ #brr.seek(0)
+ #brrdata = brr.read()
+ #lastbrroffset = None
+
+ enc = BRREncoder(pcm, None)
+ lastpcmoffset = None
+
+ # misc. variables
+ #goodrom = open('SNADNE1.667', 'rb')
+ wrong = 0
+ controlwrong = 0
+ indices = []
+
+ while str.find('PCMF') >= 0:
+ index = str.find('PCMF')
+ filepos = samplestart + index
+
+ # error checking to prevent infinite loops
+ #assert index not in indices
+ #indices.append(index)
+
+ pcmf, pcmoffset = struct.unpack('<4sI', str[index:index+8])
+ pcmoffset &= 0xffffff
+ if pcmoffset % 16 or pcmoffset < lastpcmoffset:
+ #print '%08x: unexpected offset %d' % (filepos, pcmoffset)
+ pcmoffset = lastpcmoffset + 16
+ #else:
+ # brroffset = 9 * (brroffset >> 4)
+
+ # read the BRR sample
+ #brr.seek(brroffset)
+ #brrsample = brr.read(9)
+
+ # read and encode the BRR block
+ brrsample = enc.encode_block(pcmoffset)
+
+ # error checking for invalid BRR offsets
+ if len(brrsample) != 9:
+ raise ValueError('Invalid BRR offset: %d' % brroffset)
+
+ # set the END bit in the BRR sample if it is set in the PCMF block
+ if ord(str[index+7]) & 1:
+ brrsample = chr(ord(brrsample[0]) | 1) + brrsample[1:]
+
+ # set the LOOP bit in the BRR sample if it is set in the PCMF block
+ if ord(str[index+7]) & 2:
+ brrsample = chr(ord(brrsample[0]) | 2) + brrsample[1:]
+
+ # checks whether sample matches the original ROM, when the original ROM is available (for debugging purposes)
+ '''goodrom.seek(samplestart + index)
+ grsample = goodrom.read(9)
+ if brrsample != grsample:
+ wrong += 1
+ sys.stdout.write('%08x: ' % filepos)
+ #if brrdata.find(grsample) >= 0:
+ # print 'wrong sample, correct BRR offset is %08x' % brrdata.find(grsample)
+ if brrsample[1:] == grsample[1:]:
+ if abs(ord(brrsample[0]) - ord(grsample[0])) <= 3:
+ controlwrong += 1
+ print 'SPC700 control bits differ'
+ else:
+ print 'flags are different'
+ else:
+ print 'sample encoded differently?' '''
+
+ rom.seek(index)
+ rom.write(brrsample)
+ str = rom.getvalue()
+ lastpcmoffset = pcmoffset
+
+ #print '%d wrong samples' % wrong
+ #print '%d differences in SPC700 control bits' % controlwrong
+
+ rom.close()
+ vcrom.seek(0)
+ return vcrom.read(samplestart) + str
+
+if __name__ == '__main__':
+ import time
+
+ if len(sys.argv) != 4:
+ print 'Usage: snesrestore game.rom game.pcm output.smc'
+ sys.exit(1)
+
+ vcrom = open(sys.argv[1], 'rb')
+ pcm = open(sys.argv[2], 'rb')
+
+ '''# encode raw PCM in SNES BRR format
+ print 'Encoding audio as BRR'
+ brr = StringIO()
+ enc = BRREncoder(pcm, brr)
+ enc.encode()
+ pcm.close()'''
+
+ # encode and inject BRR sound data into the ROM
+ print 'Encoding and restoring BRR audio data to ROM'
+ start = time.clock()
+ string = restore_brr_samples(vcrom, pcm)
+ end = time.clock()
+ print 'Time: %.2f seconds' % (end - start)
+
+ # write to file
+ output = open(sys.argv[3], "wb")
+ output.write(string)
+ output.close()
+ pcm.close()
+ #print 'done'
+
165 u8archive.py
@@ -0,0 +1,165 @@
+#!/usr/bin/env python
+# Author: Bryan Cain (Plombo)
+# Date: December 27, 2010
+# Description: Reads Wii U8 archives.
+
+import os, struct, posixpath
+from cStringIO import StringIO
+import lz77, huf8, lzh8
+
+class U8Archive(object):
+ # archive can be a string (filesystem path) or file-like object
+ def __init__(self, archive):
+ if type(archive) == str:
+ self.file = open(archive, 'rb')
+ else:
+ self.file = archive
+ self.files = []
+ self.readheader()
+
+ def readheader(self):
+ magic, rootnode_offset, header_size, data_offset = tuple(struct.unpack('>IIII', self.file.read(16)))
+ assert magic == 0x55aa382d
+ assert rootnode_offset == 0x20
+ assert self.file.read(16) == 16 * '\0'
+
+ root = Node(self.file, rootnode_offset)
+ root.path = '<root>'
+ path = ''
+ curdirs = [root.size]
+ dirnames = ['<root>']
+ filenum = 1
+ while curdirs:
+ node = Node(self.file, rootnode_offset + 12 * root.size)
+ node.path = posixpath.join(path, node.name)
+ filenum += 1
+ if node.type == 0x100:
+ # change current path if this is a directory
+ path = node.path
+ #print node.name, node.path
+ curdirs.append(node.size)
+ dirnames.append(node.name)
+ else: self.files.append(node)
+
+ indices = range(len(curdirs))
+ indices.reverse()
+ while curdirs and filenum >= curdirs[len(curdirs)-1]:
+ #print 'done with ' + dirnames.pop() + ' at %d' % filenum
+ path = posixpath.dirname(path)
+ curdirs.pop()
+
+ # closes the physical file associated with this archive
+ def close(self):
+ self.file.close()
+
+ # returns True if this archive has a file with the given path
+ def hasfile(self, path):
+ f = self.getfile(path)
+ if f: f.close(); return True
+ else: return False
+
+ # returns a file-like object (actually a cStringIO object) for the specified
+ # file; detects and decompresses compressed files (LZ77/Huf8/LZH8) automatically!
+ # path: file name (string) or actual file node, but NOT node path :D
+ def getfile(self, path):
+ for node in self.files:
+ if node == path or (type(path) == str and node.name.endswith(path)):
+ if node == path: path = node.name
+ self.file.seek(node.data_offset)
+ file = StringIO(self.file.read(node.size))
+ if path.startswith("LZ77"):
+ try:
+ decompressed_file = lz77.decompress(file)
+ file.close()
+ return decompressed_file
+ except ValueError, IndexError:
+ print "LZ77 decompression of '%s' failed" % path
+ print 'Dumping compressed file to %s' % path
+ f2 = open(path, "wb")
+ f2.write(file.read())
+ f2.close()
+ file.close()
+ return None
+ elif path.startswith("Huf8"):
+ try:
+ decompressed_file = StringIO()
+ huf8.decompress(file, decompressed_file)
+ file.close()
+ decompressed_file.seek(0)
+ return decompressed_file
+ except Exception:
+ print "Huf8 decompression of '%s' failed" % path
+ print "Dumping compressed file to %s" % path
+ f2 = open(path, "wb")
+ f2.write(file.read())
+ f2.close()
+ file.close()
+ return decompressed_file
+ elif path.startswith("LZH8"):
+ try:
+ decompressed_file = StringIO()
+ decompressed_file.write(lzh8.decompress(file, decompressed_file))
+ decompressed_file.seek(0)
+ file.close()
+ return decompressed_file
+ except Exception:
+ print "LZH8 decompression of '%s' failed" % path
+ print "Dumping compressed file to %s" % path
+ f2 = open(path, "wb")
+ f2.write(file.read())
+ f2.close()
+ file.close()
+ else:
+ return file
+ return None
+
+ # finds a file with the given name, accounting for compression prefixes like "LZ77", "Huf8", etc.
+ def findfile(self, name):
+ for f in self.files:
+ if f.name in (name, "LZ77"+name, "Huf8"+name, "LZH8"+name): return f.name
+ return None
+
+ def extract(self, dest):
+ if not os.path.lexists(dest): os.makedirs(dest)
+ for node in self.files:
+ if node.name in ('<root>', '.'): continue
+ if node.type == 0x100:
+ os.makedirs(os.path.join(dest, node.path))
+ #print 'created dir %s' % os.path.join(dest, node.path)
+ else:
+ #print node.path
+ path = os.path.join(dest, node.path)
+ if not os.path.lexists(os.path.dirname(path)): os.makedirs(os.path.dirname(path))
+ f = open(path, 'wb')
+ contents = self.getfile(node)
+ contents.seek(0)
+ f.write(contents.read())
+ f.close()
+ #print 'extracted file %s' % os.path.join(dest, node.path)
+
+# file node object
+class Node(object):
+ def __init__(self, arcfile, stringoffset):
+ self.rawdata = arcfile.read(12)
+ arcpos = arcfile.tell()
+ chunk1, self.data_offset, self.size = tuple(struct.unpack('>III', self.rawdata))
+ self.type = chunk1 >> 16
+ self.name_offset = chunk1 & 0xffffff
+
+ # the root node has a name_offset of 0
+ if not self.name_offset: return
+
+ # no sane file name should be more than 64 bytes; if one is, string.index() will throw an exception
+ arcfile.seek(stringoffset + self.name_offset)
+ self.name = arcfile.read(64)
+ self.name = self.name[0:self.name.index('\0')]
+ #print self.name
+ arcfile.seek(arcpos)
+
+if __name__ == '__main__':
+ # Quick functionality test and sanity check; will only work on my (Plombo's) computer without a path change
+ import os, os.path
+ arc = U8Archive(os.path.join(os.getenv('HOME'), 'wii/ssb/00000005.app'))
+ print arc.hasfile('romc')
+ print len(arc.getfile('romc').read())
+
328 wiimetadata.py
@@ -0,0 +1,328 @@
+#!/usr/bin/env python
+# Author: Bryan Cain (Plombo)
+# Date: December 27, 2010
+# Description: Reads Wii title metadata from an extracted NAND dump.
+# Thanks to Leathl for writing Wii.cs in ShowMiiWads, which was an important
+# reference in writing this program.
+
+import os, os.path, struct
+from cStringIO import StringIO
+import romc
+from u8archive import U8Archive
+from ccfarchive import CCFArchive
+from nes_rom_extract import extract_nes_rom
+from snesrestore import restore_brr_samples
+
+# rom: file-like object
+# path: string (filesystem path)
+def writerom(rom, path):
+ f = open(path, 'wb')
+ f.write(rom.read())
+ f.close()
+ rom.seek(0)
+
+class RomExtractor(object):
+ def __init__(self, id, name, channeltype, nand):
+ self.id = id
+ self.name = name
+ self.channeltype = channeltype
+ self.nand = nand
+
+ # Get proper file extension for the ROM
+ def extension(self):
+ if self.channeltype == 'Nintendo 64': return '.z64'
+ elif self.channeltype == 'Genesis': return '.gen'
+ elif self.channeltype == 'Master System': return '.sms'
+ elif self.channeltype == 'NES': return '.nes'
+ elif self.channeltype == 'SNES': return '.smc'
+ elif self.channeltype == 'TurboGrafx16': return '.pce'
+ else: return ''
+
+ def extract(self):
+ content = self.nand.path + 'title/00010001/' + self.id + '/content/'
+ rom_extracted = False
+ manual_extracted = False
+
+ for app in os.listdir(content):
+ if not app.endswith('.app'): continue
+ app = content + app
+ if self.extractrom(app): rom_extracted = True
+ if self.extractmanual(app): manual_extracted = True
+ if rom_extracted and manual_extracted: return
+
+ if rom_extracted: print 'Unable to extract manual.'
+ elif manual_extracted: print 'Unable to extract ROM.'
+ else: print 'Unable to extract ROM and manual.'
+
+ # Actually extract the ROM
+ # Currently works for almost all NES, SNES, N64, TG16, Master System, and Genesis ROMs
+ def extractrom(self, u8path):
+ if self.channeltype != 'NES':
+ try:
+ arc = U8Archive(u8path)
+ except AssertionError:
+ return False
+
+ filename = self.name + self.extension()
+ if self.channeltype == 'Nintendo 64':
+ return self.extractrom_n64(arc, filename)
+ elif self.channeltype in ('Genesis', 'Master System'):
+ return self.extractrom_sega(arc, filename)
+ elif self.channeltype == 'NES':
+ if os.path.exists(u8path):
+ f = open(u8path, 'rb')
+ rom = extract_nes_rom(f)
+ f.close()
+ if rom:
+ print 'Got ROM: %s' % filename
+ writerom(rom, filename)
+ return True
+ else: return False
+ elif self.channeltype == 'SNES':
+ return self.extractrom_snes(arc, filename)
+ elif self.channeltype == 'TurboGrafx16':
+ return self.extractrom_tg16(arc, filename)
+
+ # default if the function hasn't returned yet
+ return False
+
+ def extractrom_n64(self, arc, filename):
+ if arc.hasfile('rom'):
+ rom = arc.getfile('rom')
+ print 'Got ROM: %s' % filename
+ writerom(rom, filename)
+ return True
+ elif arc.hasfile('romc'):
+ rom = arc.getfile('romc')
+ print 'Decompressing ROM: %s (this could take a minute or two)' % filename
+ try:
+ outfile = open(filename, 'wb')
+ outfile.write(romc.decompress(rom))
+ outfile.close()
+ print 'Got ROM: %s' % filename
+ return True
+ except IndexError: # unknown compression - something besides LZSS and romchu?
+ print 'Decompression failed: unknown compression type'
+ outfile.close()
+ os.remove(filename)
+ return False
+
+ def extractrom_sega(self, arc, filename):
+ if arc.hasfile('data.ccf'):
+ ccf = CCFArchive(arc.getfile('data.ccf'))
+
+ if ccf.hasfile('config'):
+ for line in ccf.getfile('config'):
+ if line.startswith('romfile='): romname = line[len('romfile='):].strip('/\\\"\0\r\n')
+ else:
+ print 'config not found'
+ return False
+
+ if romname:
+ print 'Found ROM: %s' % romname
+ rom = ccf.find(romname)
+ writerom(rom, filename)
+ print 'Got ROM: %s' % filename
+ return True
+ else:
+ print 'ROM filename not in config'
+ return False
+
+ def extractrom_tg16(self, arc, filename):
+ config = arc.getfile('config.ini')
+ if not config:
+ print 'config.ini not found'
+ return False
+
+ path = None
+ for line in config:
+ if line.startswith('ROM='):
+ path = line[len('ROM='):].strip('/\\\"\0\r\n')
+
+ if not path:
+ print 'ROM filename not specified in config.ini'
+ return False
+
+ print 'Found ROM: %s' % path
+ rom = arc.getfile(path)
+
+ if rom:
+ writerom(rom, filename)
+ print 'Got ROM: %s' % filename
+ return True
+ else: return False
+
+ def extractrom_snes(self, arc, filename):
+ # try to find the original ROM first
+ for f in arc.files:
+ path = f.path.split('.')
+ if len(path) == 2 and path[0].startswith('SN') and path[1].isdigit():
+ print 'Found original ROM: %s' % f.path
+ rom = arc.getfile(f.path)
+ writerom(rom, filename)
+ print 'Got ROM: %s' % filename
+ return True
+
+ # if original ROM not present, try to create a playable ROM by recreating and injecting the original sounds
+ for f in arc.files:
+ path = f.path.split('.')
+ if len(path) == 2 and path[1] == 'rom':
+ print "Recreating original ROM from %s" % f.path
+ vcrom = arc.getfile(f.path)
+ if not vcrom: print "Error in reading ROM file %s" % f.path; return False
+
+ # find raw PCM data
+ pcm = None
+ for f2 in arc.files:
+ path2 = f2.path.split('.')
+ if len(path2) == 2 and path2[1] == 'pcm':
+ pcm = arc.getfile(f2.path)
+ if not pcm: print 'Error: PCM audio data not found'; return False
+
+ '''# encode raw PCM in SNES BRR format
+ print 'Encoding audio as BRR'
+ brr = StringIO()
+ enc = BRREncoder(pcm, brr)
+ enc.encode()
+ pcm.close()'''
+
+ # inject BRR audio into the ROM
+ print 'Encoding and restoring BRR audio data to ROM'
+ romdata = restore_brr_samples(vcrom, pcm)
+ vcrom.close()
+ pcm.close()
+
+ # write the recreated ROM to disk
+ f = open(filename, 'wb')
+ f.write(romdata)
+ f.close()
+ print 'Got ROM: %s' % filename
+ return True
+
+ return False
+
+ # copy save file verbatim
+ def extractsave(self, dest):
+ path = self.nand.path + 'title/00010001/' + self.id + '/data/savedata.bin'
+ if os.path.exists(path):
+ shutil.copy2(path, dest)
+ return True
+ else: return False
+
+ def extractmanual(self, u8path):
+ try:
+ arc = U8Archive(u8path)
+ except AssertionError:
+ return False
+
+ man = None
+ if arc.findfile('emanual.arc'):
+ man = U8Archive(arc.getfile(arc.findfile('emanual.arc')))
+ elif arc.findfile('html.arc'):
+ man = U8Archive(arc.getfile(arc.findfile('html.arc')))
+ elif arc.findfile('man.arc'):
+ man = U8Archive(arc.getfile(arc.findfile('man.arc')))
+ elif arc.findfile('data.ccf'):
+ ccf = CCFArchive(arc.getfile(arc.findfile('data.ccf')))
+ man = U8Archive(ccf.getfile('man.arc'))
+ elif arc.findfile('htmlc.arc'):
+ manc = arc.getfile(arc.findfile('htmlc.arc'))
+ print 'Decompressing manual: htmlc.arc'
+ man = U8Archive(StringIO(romc.decompress(manc)))
+
+ if man:
+ man.extract(os.path.join('manuals', self.name))
+ print 'Extracted manual to ' + os.path.join('manuals', self.name)
+ return True
+
+ return False
+
+class NandDump(object):
+ # path: path on filesystem to the extracted NAND dump
+ def __init__(self, path):
+ self.path = path + '/'
+
+ def scantickets(self):
+ tickets = os.listdir(self.path + '/ticket/00010001')
+ for ticket in tickets:
+ id = ticket.rstrip('.tik')
+ content = 'title/00010001/' + id + '/content/'
+ title = content + 'title.tmd'
+ if(os.path.exists(self.path + title)):
+ appname = self.getappname(title)
+ if not appname: continue
+ #print title, content + appname
+ name = self.gettitle(content + appname)
+ channeltype = self.channeltype(ticket)
+ if name and channeltype:
+ print '%s: %s' % (channeltype, name)
+ #print id
+ ext = RomExtractor(id, name, channeltype, self)
+ ext.extract()
+ print
+
+ # Returns a string denoting the channel type. Returns None if it's not a VC game.
+ def channeltype(self, ticket):
+ f = open(self.path + '/ticket/00010001/' + ticket, 'rb')
+ f.seek(0x1dc)
+ thistype = struct.unpack('>I', f.read(4))[0]
+ if thistype != 0x10001: return None
+ f.seek(0x221)
+ if struct.unpack('>B', f.read(1))[0] != 1: return None
+ f.seek(0x1e0)
+ ident = f.read(2)
+
+ # TODO: support the commented game types
+ if ident[0] == 'F': return 'NES'
+ elif ident[0] == 'J': return 'SNES'
+ elif ident[0] == 'L': return 'Master System'
+ elif ident[0] == 'M': return 'Genesis'
+ elif ident[0] == 'N': return 'Nintendo 64'
+ elif ident[0] == 'P': return 'TurboGrafx16'
+ #elif ident == 'EA': return 'Neo Geo'
+ #elif ident[0] == 'E': return 'Arcade'
+ #elif ident[0] == 'Q': return 'TurboGrafx CD'
+ #elif ident[0] == 'C': return 'Commodore 64'
+ else: return None
+
+ # Returns the path to the 00.app file containing the game's title
+ # Precondition: the file denoted by "title" exists on the filesystem
+ def getappname(self, title):
+ f = open(self.path + title, 'rb')
+ f.seek(0x1de)
+ count = struct.unpack('>H', f.read(2))[0]
+ f.seek(0x1e4)
+ appname = None
+ for i in range(count):
+ info = struct.unpack('>IHHQ', f.read(16))
+ f.read(20)
+ if info[1] == 0:
+ appname = '%08x.app' % info[0]
+ return appname
+
+ # Gets title (in English) from a 00.app file
+ def gettitle(self, path):
+ if not os.path.exists(self.path + path): return None
+ f = open(self.path + path, 'rb')
+ data = f.read()
+ f.close()
+ index = data.find('IMET')
+ if index < 0: return None
+ engindex = index + 29 + 84
+ title = data[engindex:engindex+84]
+
+ # Format the title properly
+ title = title.strip('\0')
+ while title.find('\0\0\0') >= 0: title = title.replace('\0\0\0', '\0\0')
+ title = title.replace('\0\0', ' - ')
+ title = title.replace('\0', '')
+ title = title.replace(':', ' - ')
+ while title.find(' ') >= 0: title = title.replace(' ', ' ')
+ return title
+
+if __name__ == '__main__':
+ import sys
+ nand = NandDump(sys.argv[1])
+ nand.scantickets()
+ if len(sys.argv) >= 3: print nand.gettitle(sys.argv[2])
+

0 comments on commit 8498fd4

Please sign in to comment.
Something went wrong with that request. Please try again.