Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Fixing a endian-ness bug in read_ziplist_entry.

The Redis dump file is using little endian numbers everywhere except for the
length of ziplist entries of more than 16,383 bytes. That is a hard edge case
to trigger, because it requires increasing hash-max-ziplist-value, since the
default insures that this case will never happen. I've added a test to catch
the issue, but it requires setting hash-max-ziplist-value to over 20,000.
  • Loading branch information...
commit 4a4b7e073aae64a5ec96dc461a25983843f09ea1 1 parent d48d74b
Charles Gordon authored
View
5 rdbtools/parser.py
@@ -551,7 +551,7 @@ def read_ziplist_entry(self, f) :
length = ((entry_header & 0x3F) << 8) | read_unsigned_char(f)
value = f.read(length)
elif (entry_header >> 6) == 2 :
- length = read_unsigned_int(f)
+ length = read_big_endian_unsigned_int(f)
value = f.read(length)
elif (entry_header >> 4) == 12 :
value = read_signed_short(f)
@@ -724,6 +724,9 @@ def read_signed_int(f) :
def read_unsigned_int(f) :
return struct.unpack('I', f.read(4))[0]
+def read_big_endian_unsigned_int(f):
+ return struct.unpack('>I', f.read(4))[0]
+
def read_24bit_signed_number(f):
s = '0' + f.read(3)
num = struct.unpack('i', s)[0]
View
BIN  tests/dumps/zipmap_with_big_values.rdb
Binary file not shown
View
6 tests/parser_tests.py
@@ -67,12 +67,18 @@ def test_zipmap_with_big_values(self):
''' See issue https://github.com/sripathikrishnan/redis-rdb-tools/issues/2
Values with length around 253/254/255 bytes are treated specially in the parser
This test exercises those boundary conditions
+
+ In order to test a bug with large ziplists, it is necessary to start
+ Redis with "hash-max-ziplist-value 21000", create this rdb file,
+ and run the test. That forces the 20kbyte value to be stored as a
+ ziplist with a length encoding of 5 bytes.
'''
r = load_rdb('zipmap_with_big_values.rdb')
self.assertEquals(len(r.databases[0]["zipmap_with_big_values"]["253bytes"]), 253)
self.assertEquals(len(r.databases[0]["zipmap_with_big_values"]["254bytes"]), 254)
self.assertEquals(len(r.databases[0]["zipmap_with_big_values"]["255bytes"]), 255)
self.assertEquals(len(r.databases[0]["zipmap_with_big_values"]["300bytes"]), 300)
+ self.assertEquals(len(r.databases[0]["zipmap_with_big_values"]["20kbytes"]), 20000)
def test_hash_as_ziplist(self):
'''In redis dump version = 4, hashmaps are stored as ziplists'''
Please sign in to comment.
Something went wrong with that request. Please try again.