Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Issue #6 Support RDB version 6

Version 6 adds special encoding for integers in ziplists
- Numbers 0 through 12 are encoded within the entry headeer
- There is a special encoding for 8 bit and 24 bit signed integers
  • Loading branch information...
commit 13e5e76d6e35040629fd302d070960acee2c6eca 1 parent a3db2e6
@sripathikrishnan authored
View
13 rdbtools/parser.py
@@ -559,6 +559,12 @@ def read_ziplist_entry(self, f) :
value = read_signed_int(f)
elif (entry_header >> 4) == 14 :
value = read_signed_long(f)
+ elif (entry_header == 240) :
+ value = read_24bit_signed_number(f)
+ elif (entry_header == 254) :
+ value = read_signed_char(f)
+ elif (entry_header >= 241 and entry_header <= 253) :
+ value = entry_header - 241
else :
raise Exception('read_ziplist_entry', 'Invalid entry_header %d for key %s' % (entry_header, self._key))
return value
@@ -602,7 +608,7 @@ def verify_magic_string(self, magic_string) :
def verify_version(self, version_str) :
version = int(version_str)
- if version < 1 or version > 5 :
+ if version < 1 or version > 6 :
raise Exception('verify_version', 'Invalid RDB version number %d' % version)
def init_filter(self, filters):
@@ -718,6 +724,11 @@ def read_signed_int(f) :
def read_unsigned_int(f) :
return struct.unpack('I', f.read(4))[0]
+def read_24bit_signed_number(f):
+ s = '0' + f.read(3)
+ num = struct.unpack('i', s)[0]
+ return num >> 8
+
def read_signed_long(f) :
return struct.unpack('q', f.read(8))[0]
View
60 tests/create_test_rdb.py
@@ -10,26 +10,26 @@
def create_test_rdbs(path_to_redis_dump, dump_folder) :
clean_database()
tests = (
- empty_database,
- multiple_databases,
- keys_with_expiry,
- integer_keys,
- uncompressible_string_keys,
- easily_compressible_string_key,
- zipmap_that_doesnt_compress,
- zipmap_that_compresses_easily,
- zipmap_with_big_values,
- dictionary,
- ziplist_that_compresses_easily,
- ziplist_that_doesnt_compress,
+# empty_database,
+# multiple_databases,
+# keys_with_expiry,
+# integer_keys,
+# uncompressible_string_keys,
+# easily_compressible_string_key,
+# zipmap_that_doesnt_compress,
+# zipmap_that_compresses_easily,
+# zipmap_with_big_values,
+# dictionary,
+# ziplist_that_compresses_easily,
+# ziplist_that_doesnt_compress,
ziplist_with_integers,
- linkedlist,
- intset_16,
- intset_32,
- intset_64,
- regular_set,
- sorted_set_as_ziplist,
- regular_sorted_set
+# linkedlist,
+# intset_16,
+# intset_32,
+# intset_64,
+# regular_set,
+# sorted_set_as_ziplist,
+# regular_sorted_set
)
for t in tests :
create_rdb_file(t, path_to_redis_dump, dump_folder)
@@ -103,11 +103,31 @@ def ziplist_that_doesnt_compress() :
r.rpush("ziplist_doesnt_compress", "cc953a17a8e096e76a44169ad3f9ac87c5f8248a403274416179aa9fbd852344")
def ziplist_with_integers() :
+
+ # Integers between 0 and 12, both inclusive, are encoded differently
+ for x in range(0,13):
+ r.rpush("ziplist_with_integers", x)
+
+
+ # Dealing with 1 byte integers
+ r.rpush("ziplist_with_integers", -2)
+ r.rpush("ziplist_with_integers", 13)
+ r.rpush("ziplist_with_integers", 25)
+ r.rpush("ziplist_with_integers", -61)
r.rpush("ziplist_with_integers", 63)
+
+ # Dealing with 2 byte integers
r.rpush("ziplist_with_integers", 16380)
+ r.rpush("ziplist_with_integers", -16000)
+
+ # Dealing with 4 byte signed integers
r.rpush("ziplist_with_integers", 65535)
- r.rpush("ziplist_with_integers", 0x7fffffffffffffff)
+ r.rpush("ziplist_with_integers", -65523)
+ # Dealing with 8 byte signed integers
+ r.rpush("ziplist_with_integers", 4194304)
+ r.rpush("ziplist_with_integers", 0x7fffffffffffffff)
+
def linkedlist() :
num_entries = 1000
for x in xrange(0, num_entries) :
View
BIN  tests/dumps/ziplist_with_integers.rdb
Binary file not shown
View
14 tests/parser_tests.py
@@ -104,9 +104,17 @@ def test_ziplist_that_doesnt_compress(self):
def test_ziplist_with_integers(self):
r = load_rdb('ziplist_with_integers.rdb')
- self.assertEquals(r.lengths[0]["ziplist_with_integers"], 4)
- for num in (63, 16380, 65535, 0x7fffffffffffffff) :
- self.assert_(num in r.databases[0]["ziplist_with_integers"])
+
+ expected_numbers = []
+ for x in range(0,13):
+ expected_numbers.append(x)
+
+ expected_numbers += [-2, 13, 25, -61, 63, 16380, -16000, 65535, -65523, 4194304, 0x7fffffffffffffff]
+
+ self.assertEquals(r.lengths[0]["ziplist_with_integers"], len(expected_numbers))
+
+ for num in expected_numbers :
+ self.assert_(num in r.databases[0]["ziplist_with_integers"], "Cannot find %d" % num)
def test_linkedlist(self):
r = load_rdb('linkedlist.rdb')
Please sign in to comment.
Something went wrong with that request. Please try again.