Permalink
Browse files

Merge pull request #16 from cgordon/master

rdb-parser fails for ziplists when an entry is more than 16383 bytes - Wrong Endian Format
  • Loading branch information...
2 parents d48d74b + 3e6b705 commit b44e2100ed7e185e8f208b1601eda42807b3c2af @sripathikrishnan committed Oct 4, 2012
View
@@ -551,7 +551,7 @@ def read_ziplist_entry(self, f) :
length = ((entry_header & 0x3F) << 8) | read_unsigned_char(f)
value = f.read(length)
elif (entry_header >> 6) == 2 :
- length = read_unsigned_int(f)
+ length = read_big_endian_unsigned_int(f)
value = f.read(length)
elif (entry_header >> 4) == 12 :
value = read_signed_short(f)
@@ -724,6 +724,9 @@ def read_signed_int(f) :
def read_unsigned_int(f) :
return struct.unpack('I', f.read(4))[0]
+def read_big_endian_unsigned_int(f):
+ return struct.unpack('>I', f.read(4))[0]
+
def read_24bit_signed_number(f):
s = '0' + f.read(3)
num = struct.unpack('i', s)[0]
View
@@ -88,6 +88,7 @@ def zipmap_with_big_values():
r.hset("zipmap_with_big_values", "254bytes", random_string(254, 'seed2'))
r.hset("zipmap_with_big_values", "255bytes", random_string(255, 'seed3'))
r.hset("zipmap_with_big_values", "300bytes", random_string(300, 'seed4'))
+ r.hset("zipmap_with_big_values", "20kbytes", random_string(20000, 'seed5'))
def dictionary() :
num_entries = 1000
Binary file not shown.
View
@@ -67,12 +67,18 @@ def test_zipmap_with_big_values(self):
''' See issue https://github.com/sripathikrishnan/redis-rdb-tools/issues/2
Values with length around 253/254/255 bytes are treated specially in the parser
This test exercises those boundary conditions
+
+ In order to test a bug with large ziplists, it is necessary to start
+ Redis with "hash-max-ziplist-value 21000", create this rdb file,
+ and run the test. That forces the 20kbyte value to be stored as a
+ ziplist with a length encoding of 5 bytes.
'''
r = load_rdb('zipmap_with_big_values.rdb')
self.assertEquals(len(r.databases[0]["zipmap_with_big_values"]["253bytes"]), 253)
self.assertEquals(len(r.databases[0]["zipmap_with_big_values"]["254bytes"]), 254)
self.assertEquals(len(r.databases[0]["zipmap_with_big_values"]["255bytes"]), 255)
self.assertEquals(len(r.databases[0]["zipmap_with_big_values"]["300bytes"]), 300)
+ self.assertEquals(len(r.databases[0]["zipmap_with_big_values"]["20kbytes"]), 20000)
def test_hash_as_ziplist(self):
'''In redis dump version = 4, hashmaps are stored as ziplists'''

0 comments on commit b44e210

Please sign in to comment.