Skip to content
This repository
Browse code

Merge pull request #16 from cgordon/master

rdb-parser fails for ziplists when an entry is more than 16383 bytes - Wrong Endian Format
  • Loading branch information...
commit b44e2100ed7e185e8f208b1601eda42807b3c2af 2 parents d48d74b + 3e6b705
Sripathi Krishnan authored
5  rdbtools/parser.py
@@ -551,7 +551,7 @@ def read_ziplist_entry(self, f) :
551 551
             length = ((entry_header & 0x3F) << 8) | read_unsigned_char(f)
552 552
             value = f.read(length)
553 553
         elif (entry_header >> 6) == 2 :
554  
-            length = read_unsigned_int(f)
  554
+            length = read_big_endian_unsigned_int(f)
555 555
             value = f.read(length)
556 556
         elif (entry_header >> 4) == 12 :
557 557
             value = read_signed_short(f)
@@ -724,6 +724,9 @@ def read_signed_int(f) :
724 724
 def read_unsigned_int(f) :
725 725
     return struct.unpack('I', f.read(4))[0]
726 726
 
  727
+def read_big_endian_unsigned_int(f):
  728
+    return struct.unpack('>I', f.read(4))[0]
  729
+
727 730
 def read_24bit_signed_number(f):
728 731
     s = '0' + f.read(3)
729 732
     num = struct.unpack('i', s)[0]
1  tests/create_test_rdb.py
@@ -88,6 +88,7 @@ def zipmap_with_big_values():
88 88
     r.hset("zipmap_with_big_values", "254bytes", random_string(254, 'seed2'))
89 89
     r.hset("zipmap_with_big_values", "255bytes", random_string(255, 'seed3'))
90 90
     r.hset("zipmap_with_big_values", "300bytes", random_string(300, 'seed4'))
  91
+    r.hset("zipmap_with_big_values", "20kbytes", random_string(20000, 'seed5'))
91 92
     
92 93
 def dictionary() :
93 94
     num_entries = 1000
BIN  tests/dumps/zipmap_with_big_values.rdb
Binary file not shown
6  tests/parser_tests.py
@@ -67,12 +67,18 @@ def test_zipmap_with_big_values(self):
67 67
         ''' See issue https://github.com/sripathikrishnan/redis-rdb-tools/issues/2
68 68
             Values with length around 253/254/255 bytes are treated specially in the parser
69 69
             This test exercises those boundary conditions
  70
+
  71
+            In order to test a bug with large ziplists, it is necessary to start
  72
+            Redis with "hash-max-ziplist-value 21000", create this rdb file,
  73
+            and run the test. That forces the 20kbyte value to be stored as a
  74
+            ziplist with a length encoding of 5 bytes.
70 75
         '''
71 76
         r = load_rdb('zipmap_with_big_values.rdb')
72 77
         self.assertEquals(len(r.databases[0]["zipmap_with_big_values"]["253bytes"]), 253)
73 78
         self.assertEquals(len(r.databases[0]["zipmap_with_big_values"]["254bytes"]), 254)
74 79
         self.assertEquals(len(r.databases[0]["zipmap_with_big_values"]["255bytes"]), 255)
75 80
         self.assertEquals(len(r.databases[0]["zipmap_with_big_values"]["300bytes"]), 300)
  81
+        self.assertEquals(len(r.databases[0]["zipmap_with_big_values"]["20kbytes"]), 20000)
76 82
         
77 83
     def test_hash_as_ziplist(self):
78 84
         '''In redis dump version = 4, hashmaps are stored as ziplists'''

0 notes on commit b44e210

Please sign in to comment.
Something went wrong with that request. Please try again.