COMMON: Use a prefix table to speed up the Huffman decoder

Symbols for codes shorter than the prefix table index width are stored
in the table. All the entries in the table with an index starting with
the code are set to the symbol value. That way, when decoding it is
possible to get the number of bits corresponding to the table width from
the bitstream and directly find the symbol value. Longer code still need
to be searched for in the codes list.

common/huffman.h

@@ -20,7 +20,7 @@
  *
  */
 
-// Based on eos' Huffman code
+// Based on xoreos' Huffman code
 
 #ifndef COMMON_HUFFMAN_H
 #define COMMON_HUFFMAN_H
@@ -31,12 +31,22 @@
 
 namespace Common {
 
+inline uint32 REVERSEBITS(uint32 x) {
+	x = (((x & ~0x55555555) >> 1) | ((x & 0x55555555) << 1));
+	x = (((x & ~0x33333333) >> 2) | ((x & 0x33333333) << 2));
+	x = (((x & ~0x0F0F0F0F) >> 4) | ((x & 0x0F0F0F0F) << 4));
+	x = (((x & ~0x00FF00FF) >> 8) | ((x & 0x00FF00FF) << 8));
+
+	return((x >> 16) | (x << 16));
+}
+
 /**
  * Huffman bitstream decoding
  *
  * Used in engines:
  *  - scumm
  */
+template<class BITSTREAM>
 class Huffman {
 public:
 	/** Construct a Huffman decoder.
@@ -48,47 +58,107 @@ class Huffman {
 	 *  @param symbols The symbols. If 0, assume they are identical to the code indices.
 	 */
 	Huffman(uint8 maxLength, uint32 codeCount, const uint32 *codes, const uint8 *lengths, const uint32 *symbols = nullptr);
-	~Huffman();
-
-	/** Modify the codes' symbols. */
-	void setSymbols(const uint32 *symbols = nullptr);
 
 	/** Return the next symbol in the bitstream. */
-	template<class BITSTREAM>
-	uint32 getSymbol(BITSTREAM &bits) const {
-		uint32 code = 0;
-
-		for (uint32 i = 0; i < _codes.size(); i++) {
-			bits.addBit(code, i);
-
-			for (CodeList::const_iterator cCode = _codes[i].begin(); cCode != _codes[i].end(); ++cCode)
-				if (code == cCode->code)
-					return cCode->symbol;
-		}
-
-		error("Unknown Huffman code");
-		return 0;
-	}
+	uint32 getSymbol(BITSTREAM &bits) const;
 
 private:
 	struct Symbol {
 		uint32 code;
 		uint32 symbol;
 
-		Symbol(uint32 c, uint32 s);
+		Symbol(uint32 c, uint32 s) : code(c), symbol(s) {}
 	};
 
 	typedef List<Symbol> CodeList;
 	typedef Array<CodeList> CodeLists;
-	typedef Array<Symbol *> SymbolList;
 
 	/** Lists of codes and their symbols, sorted by code length. */
 	CodeLists _codes;
 
-	/** Sorted list of pointers to the symbols. */
-	SymbolList _symbols;
+	/** Prefix lookup table used to speed up the decoding of short codes. */
+	struct PrefixEntry {
+		uint32 symbol;
+		uint8  length;
+
+		PrefixEntry() : length(0xFF) {}
+	};
+
+	static const uint8 _prefixTableBits = 8;
+	PrefixEntry _prefixTable[1 << _prefixTableBits];
 };
 
+template <class BITSTREAM>
+Huffman<BITSTREAM>::Huffman(uint8 maxLength, uint32 codeCount, const uint32 *codes, const uint8 *lengths, const uint32 *symbols) {
+	assert(codeCount > 0);
+
+	assert(codes);
+	assert(lengths);
+
+	if (maxLength == 0)
+		for (uint32 i = 0; i < codeCount; i++)
+			maxLength = MAX(maxLength, lengths[i]);
+
+	assert(maxLength <= 32);
+
+	// Codes that don't fit in the prefix table are stored in the _codes array
+	_codes.resize(MAX(maxLength - _prefixTableBits, 0));
+
+	for (uint i = 0; i < codeCount; i++) {
+		uint8 length = lengths[i];
+
+		// The symbol. If none were specified, just assume it's identical to the code index
+		uint32 symbol = symbols ? symbols[i] : i;
+
+		if (length <= _prefixTableBits) {
+			// Short codes go in the prefix lookup table. Set all the entries in the table
+			// with an index starting with the code to the symbol value.
+			uint32 startIndex;
+			if (BITSTREAM::isMSB2LSB()) {
+				startIndex = codes[i] << (_prefixTableBits - length);
+			} else {
+				startIndex = REVERSEBITS(codes[i]) >> (32 - _prefixTableBits);
+			}
+
+			uint32 endIndex = startIndex | ((1 << (_prefixTableBits - length)) - 1);
+
+			for (uint32 j = startIndex; j <= endIndex; j++) {
+				uint32 index = BITSTREAM::isMSB2LSB() ? j : REVERSEBITS(j) >> (32 - _prefixTableBits);
+				_prefixTable[index].symbol = symbol;
+				_prefixTable[index].length = length;
+			}
+		} else {
+			// Put the code and symbol into the correct list for the length
+			_codes[lengths[i] - 1 - _prefixTableBits].push_back(Symbol(codes[i], symbol));
+		}
+	}
+}
+
+template <class BITSTREAM>
+uint32 Huffman<BITSTREAM>::getSymbol(BITSTREAM &bits) const {
+	uint32 code = bits.peekBits(_prefixTableBits);
+
+	uint8 length = _prefixTable[code].length;
+
+	if (length != 0xFF) {
+		bits.skip(length);
+		return _prefixTable[code].symbol;
+	} else {
+		bits.skip(_prefixTableBits);
+
+		for (uint32 i = 0; i < _codes.size(); i++) {
+			bits.addBit(code, i + _prefixTableBits);
+
+			for (typename CodeList::const_iterator cCode = _codes[i].begin(); cCode != _codes[i].end(); ++cCode)
+				if (code == cCode->code)
+					return cCode->symbol;
+		}
+	}
+
+	error("Unknown Huffman code");
+	return 0;
+}
+
 } // End of namespace Common
 
 #endif // COMMON_HUFFMAN_H

common/module.mk

@@ -49,7 +49,6 @@ MODULE_OBJS += \
 	cosinetables.o \
 	dct.o \
 	fft.o \
-	huffman.o \
 	rdft.o \
 	sinetables.o
 

image/codecs/svq1.cpp

@@ -56,16 +56,16 @@ SVQ1Decoder::SVQ1Decoder(uint16 width, uint16 height) {
 	_last[2] = 0;
 
 	// Setup Variable Length Code Tables
-	_blockType = new Common::Huffman(0, 4, s_svq1BlockTypeCodes, s_svq1BlockTypeLengths);
+	_blockType = new HuffmanDecoder(0, 4, s_svq1BlockTypeCodes, s_svq1BlockTypeLengths);
 
 	for (int i = 0; i < 6; i++) {
-		_intraMultistage[i] = new Common::Huffman(0, 8, s_svq1IntraMultistageCodes[i], s_svq1IntraMultistageLengths[i]);
-		_interMultistage[i] = new Common::Huffman(0, 8, s_svq1InterMultistageCodes[i], s_svq1InterMultistageLengths[i]);
+		_intraMultistage[i] = new HuffmanDecoder(0, 8, s_svq1IntraMultistageCodes[i], s_svq1IntraMultistageLengths[i]);
+		_interMultistage[i] = new HuffmanDecoder(0, 8, s_svq1InterMultistageCodes[i], s_svq1InterMultistageLengths[i]);
 	}
 
-	_intraMean = new Common::Huffman(0, 256, s_svq1IntraMeanCodes, s_svq1IntraMeanLengths);
-	_interMean = new Common::Huffman(0, 512, s_svq1InterMeanCodes, s_svq1InterMeanLengths);
-	_motionComponent = new Common::Huffman(0, 33, s_svq1MotionComponentCodes, s_svq1MotionComponentLengths);
+	_intraMean = new HuffmanDecoder(0, 256, s_svq1IntraMeanCodes, s_svq1IntraMeanLengths);
+	_interMean = new HuffmanDecoder(0, 512, s_svq1InterMeanCodes, s_svq1InterMeanLengths);
+	_motionComponent = new HuffmanDecoder(0, 33, s_svq1MotionComponentCodes, s_svq1MotionComponentLengths);
 }
 
 SVQ1Decoder::~SVQ1Decoder() {

image/codecs/svq1.h

@@ -27,6 +27,7 @@
 #include "image/codecs/codec.h"
 
 namespace Common {
+template <class BITSTREAM>
 class Huffman;
 struct Point;
 }
@@ -53,12 +54,14 @@ class SVQ1Decoder : public Codec {
 
 	byte *_last[3];
 
-	Common::Huffman *_blockType;
-	Common::Huffman *_intraMultistage[6];
-	Common::Huffman *_interMultistage[6];
-	Common::Huffman *_intraMean;
-	Common::Huffman *_interMean;
-	Common::Huffman *_motionComponent;
+	typedef Common::Huffman<Common::BitStream32BEMSB> HuffmanDecoder;
+
+	HuffmanDecoder *_blockType;
+	HuffmanDecoder *_intraMultistage[6];
+	HuffmanDecoder *_interMultistage[6];
+	HuffmanDecoder *_intraMean;
+	HuffmanDecoder *_interMean;
+	HuffmanDecoder *_motionComponent;
 
 	bool svq1DecodeBlockIntra(Common::BitStream32BEMSB *s, byte *pixels, int pitch);
 	bool svq1DecodeBlockNonIntra(Common::BitStream32BEMSB *s, byte *pixels, int pitch);

test/common/huffman.h

@@ -32,7 +32,7 @@ class HuffmanTestSuite : public CxxTest::TestSuite {
 		const uint32 codes[]  = {0x2, 0x3, 0x3, 0x0, 0x2};
 		const uint32 symbols[]  = {0xA, 0xB, 0xC, 0xD, 0xE};
 
-		Common::Huffman h(maxLength, codeCount, codes, lengths, symbols);
+		Common::Huffman<Common::BitStream8MSB> h(maxLength, codeCount, codes, lengths, symbols);
 
 		byte input[] = {0x4F, 0x20};
 		// Provided input...
@@ -78,7 +78,7 @@ class HuffmanTestSuite : public CxxTest::TestSuite {
 		const uint8 lengths[] = {3,3,2,2,2};
 		const uint32 codes[]  = {0x2, 0x3, 0x3, 0x0, 0x2};
 
-		Common::Huffman h(0, codeCount, codes, lengths, 0);
+		Common::Huffman<Common::BitStream8MSB> h(0, codeCount, codes, lengths, 0);
 
 		byte input[] = {0x4F, 0x20};
 		uint32 expected[] = {0, 1, 2, 3, 4, 3 ,3};
@@ -94,51 +94,4 @@ class HuffmanTestSuite : public CxxTest::TestSuite {
 		TS_ASSERT_EQUALS(h.getSymbol(bs), expected[5]);
 		TS_ASSERT_EQUALS(h.getSymbol(bs), expected[6]);
 	}
-
-	void test_get_after_set_symbols() {
-
-		/*
-		 * Another variation of test_get_with_full_symbols.
-		 * I use the setSymbols method to define, a posteriori,
-		 * an alphabet to be used in place of array indices.
-		 * The encoding is, at first,
-		 * 0=010
-		 * 1=011
-		 * 2=11
-		 * 3=00
-		 * 4=10
-		 * (=array indices).
-		 */
-
-		uint32 codeCount = 5;
-		const uint8 lengths[] = {3,3,2,2,2};
-		const uint32 codes[]  = {0x2, 0x3, 0x3, 0x0, 0x2};
-
-		Common::Huffman h(0, codeCount, codes, lengths, 0);
-
-		const uint32 symbols[]  = {0xA, 0xB, 0xC, 0xD, 0xE};
-		h.setSymbols(symbols);
-
-		byte input[] = {0x4F, 0x20};
-		uint32 expected[] = {0xA, 0xB, 0xC, 0xD, 0xE, 0xD, 0xD};
-
-		Common::MemoryReadStream ms(input, sizeof(input));
-		Common::BitStream8MSB bs(ms);
-
-		/* New symbols:
-		 * A=010
-		 * B=011
-		 * C=11
-		 * D=00
-		 * E=10
-		 */
-
-		TS_ASSERT_EQUALS(h.getSymbol(bs), expected[0]);
-		TS_ASSERT_EQUALS(h.getSymbol(bs), expected[1]);
-		TS_ASSERT_EQUALS(h.getSymbol(bs), expected[2]);
-		TS_ASSERT_EQUALS(h.getSymbol(bs), expected[3]);
-		TS_ASSERT_EQUALS(h.getSymbol(bs), expected[4]);
-		TS_ASSERT_EQUALS(h.getSymbol(bs), expected[5]);
-		TS_ASSERT_EQUALS(h.getSymbol(bs), expected[6]);
-	}
 };

video/bink_decoder.cpp

@@ -594,7 +594,7 @@ void BinkDecoder::BinkVideoTrack::deinitBundles() {
 
 void BinkDecoder::BinkVideoTrack::initHuffman() {
 	for (int i = 0; i < 16; i++)
-		_huffman[i] = new Common::Huffman(binkHuffmanLengths[i][15], 16, binkHuffmanCodes[i], binkHuffmanLengths[i]);
+		_huffman[i] = new Common::Huffman<Common::BitStream32LELSB>(binkHuffmanLengths[i][15], 16, binkHuffmanCodes[i], binkHuffmanLengths[i]);
 }
 
 byte BinkDecoder::BinkVideoTrack::getHuffmanSymbol(VideoFrame &video, Huffman &huffman) {

video/bink_decoder.h

@@ -46,6 +46,7 @@ class QueuingAudioStream;
 
 namespace Common {
 class SeekableReadStream;
+template <class BITSTREAM>
 class Huffman;
 
 class RDFT;
@@ -247,7 +248,7 @@ class BinkDecoder : public VideoDecoder {
 
 		Bundle _bundles[kSourceMAX]; ///< Bundles for decoding all data types.
 
-		Common::Huffman *_huffman[16]; ///< The 16 Huffman codebooks used in Bink decoding.
+		Common::Huffman<Common::BitStream32LELSB> *_huffman[16]; ///< The 16 Huffman codebooks used in Bink decoding.
 
 		/** Huffman codebooks to use for decoding high nibbles in color data types. */
 		Huffman _colHighHuffman[16];