From de5131f345893dc67fac8ccad7af1b5924cd518d Mon Sep 17 00:00:00 2001 From: santialvarezcolombo Date: Wed, 13 Jan 2016 14:26:24 +0100 Subject: [PATCH 01/24] first changes to add antidote comparator --- c_src/antidote.cc | 73 +++++++++++++++++++++++++++++++++++++++++++++++ c_src/antidote.h | 11 +++++++ c_src/eleveldb.cc | 10 +++++++ src/eleveldb.erl | 3 +- 4 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 c_src/antidote.cc create mode 100644 c_src/antidote.h diff --git a/c_src/antidote.cc b/c_src/antidote.cc new file mode 100644 index 00000000..23fa05c9 --- /dev/null +++ b/c_src/antidote.cc @@ -0,0 +1,73 @@ +#include +#include +#include "leveldb/comparator.h" +#include "leveldb/slice.h" +#include "port/port.h" +#include "util/logging.h" +#include +using namespace std; + +namespace leveldb { + + //Comparator::~Comparator() { } + + namespace { + + class AntidoteComparator : public Comparator { + protected: + + public: + AntidoteComparator() { } + + virtual const char* Name() const { + return "AntidoteComparator"; + } + + virtual int Compare(const Slice& a, const Slice& b) const { + if(a == b) { + return 0; + } + + Slice ac = Slice(a.data());//, bc = Slice(b.data()); + + //Slice aname = Get32PrefData(ac), bname = Get32PrefData(bc); + + //int set_cmp = aname.compare(bname); + + cout << ac.ToString() << endl; + + return 1; + } + + // No need to shorten keys since it's fixed size. + virtual void FindShortestSeparator(std::string* start, + const Slice& limit) const { + } + + // No need to shorten keys since it's fixed size. + virtual void FindShortSuccessor(std::string* key) const { + } + + + }; + + } + + static port::OnceType antidote_once = LEVELDB_ONCE_INIT; + static const Comparator* antidote_cmp = NULL; + + static void InitAntidoteComparator() { + antidote_cmp = new AntidoteComparator(); + } + + const Comparator* GetAntidoteComparator() { + port::InitOnce(&antidote_once, InitAntidoteComparator); + return antidote_cmp; + } + + void AntidoteComparatorShutdown() { + delete antidote_cmp; + antidote_cmp = NULL; + } + +} //namespace antidote diff --git a/c_src/antidote.h b/c_src/antidote.h new file mode 100644 index 00000000..e9b4d2d7 --- /dev/null +++ b/c_src/antidote.h @@ -0,0 +1,11 @@ +#ifndef ANTIDOTE_COMPARATOR_H_ +#define ANTIDOTE_COMPARATOR_H_ + +#include "leveldb/comparator.h" + +namespace leveldb { + + extern const Comparator* GetAntidoteComparator(); +} + +#endif diff --git a/c_src/eleveldb.cc b/c_src/eleveldb.cc index 8ea4fab1..05cd2ddd 100644 --- a/c_src/eleveldb.cc +++ b/c_src/eleveldb.cc @@ -33,6 +33,7 @@ #include #include "eleveldb.h" +#include "antidote.h" #include "leveldb/db.h" #include "leveldb/comparator.h" @@ -132,6 +133,7 @@ ERL_NIF_TERM ATOM_DELETE_THRESHOLD; ERL_NIF_TERM ATOM_TIERED_SLOW_LEVEL; ERL_NIF_TERM ATOM_TIERED_FAST_PREFIX; ERL_NIF_TERM ATOM_TIERED_SLOW_PREFIX; +ERL_NIF_TERM ATOM_ANTIDOTE; } // namespace eleveldb @@ -452,6 +454,13 @@ ERL_NIF_TERM parse_open_option(ErlNifEnv* env, ERL_NIF_TERM item, leveldb::Optio } } + else if (option[0] == eleveldb::ATOM_ANTIDOTE) + { + if (option[1] == eleveldb::ATOM_TRUE) + { + opts.comparator = leveldb::GetAntidoteComparator(); + } + } return eleveldb::ATOM_OK; } @@ -1260,6 +1269,7 @@ try ATOM(eleveldb::ATOM_TIERED_SLOW_LEVEL, "tiered_slow_level"); ATOM(eleveldb::ATOM_TIERED_FAST_PREFIX, "tiered_fast_prefix"); ATOM(eleveldb::ATOM_TIERED_SLOW_PREFIX, "tiered_slow_prefix"); + ATOM(eleveldb::ATOM_ANTIDOTE, "antidote"); #undef ATOM diff --git a/src/eleveldb.erl b/src/eleveldb.erl index 1e72bab9..728241c7 100644 --- a/src/eleveldb.erl +++ b/src/eleveldb.erl @@ -102,7 +102,8 @@ init() -> {delete_threshold, pos_integer()} | {tiered_slow_level, pos_integer()} | {tiered_fast_prefix, string()} | - {tiered_slow_prefix, string()}]. + {tiered_slow_prefix, string()} | + {antidote, boolean()}]. -type read_options() :: [{verify_checksums, boolean()} | {fill_cache, boolean()} | From 64a3474014083caa40ea8621b84d003be5329a4a Mon Sep 17 00:00:00 2001 From: santialvarezcolombo Date: Wed, 13 Jan 2016 15:23:28 +0100 Subject: [PATCH 02/24] fix elseif order --- c_src/eleveldb.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/c_src/eleveldb.cc b/c_src/eleveldb.cc index 05cd2ddd..f2bc7b49 100644 --- a/c_src/eleveldb.cc +++ b/c_src/eleveldb.cc @@ -452,15 +452,14 @@ ERL_NIF_TERM parse_open_option(ErlNifEnv* env, ERL_NIF_TERM item, leveldb::Optio if (0 Date: Sun, 17 Jan 2016 17:15:41 +0100 Subject: [PATCH 03/24] added erlang external format checks + comparison of antidote keys --- c_src/antidote.cc | 39 ++++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/c_src/antidote.cc b/c_src/antidote.cc index 23fa05c9..0d4068b1 100644 --- a/c_src/antidote.cc +++ b/c_src/antidote.cc @@ -28,15 +28,44 @@ namespace leveldb { return 0; } - Slice ac = Slice(a.data());//, bc = Slice(b.data()); + Slice ac = Slice(a.data()), bc = Slice(b.data()); - //Slice aname = Get32PrefData(ac), bname = Get32PrefData(bc); + // Trim Slices and compare Antidote keys (atoms) + int aKeySize = checkAndTrimFirstBytes(ac); + int bKeySize = checkAndTrimFirstBytes(bc); - //int set_cmp = aname.compare(bname); + Slice aKey = Slice(ac.data(), aKeySize); + Slice bKey = Slice(bc.data(), bKeySize); - cout << ac.ToString() << endl; + return aKey.compare(bKey); + } + + // Given a slice, checks that the first bytes match Erlang + // external format + Antidote key, which starts with an atom. + // Returns the size of the atom to read. + static int checkAndTrimFirstBytes(Slice &s) { + // External Term Format -> first byte = 131 + assert(s[0] == (char) 131); + s.remove_prefix(1); + + // SMALL_TUPLE_EXT = 104 + assert(s[0] == (char) 104); + s.remove_prefix(1); + + // ELEMENTS in tuple = not checked for now + // assert(res[0] == (char) 1); + s.remove_prefix(1); + + // ATOM_EXT = 100 + assert(s[0] == (char) 100); + s.remove_prefix(1); + + // LENGTH of key + Slice sc = Slice(s.data(), 2); + s.remove_prefix(2); + sc.remove_prefix(1); - return 1; + return (int) sc[0]; } // No need to shorten keys since it's fixed size. From c5da7d0fd0b307a3a61667a382d771c75c49cc2b Mon Sep 17 00:00:00 2001 From: santialvarezcolombo Date: Sun, 17 Jan 2016 17:23:55 +0100 Subject: [PATCH 04/24] fix bug in Slice size while copying --- c_src/antidote.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/c_src/antidote.cc b/c_src/antidote.cc index 0d4068b1..40e3f522 100644 --- a/c_src/antidote.cc +++ b/c_src/antidote.cc @@ -28,7 +28,7 @@ namespace leveldb { return 0; } - Slice ac = Slice(a.data()), bc = Slice(b.data()); + Slice ac = Slice(a.data(), a.size()), bc = Slice(b.data(), b.size()); // Trim Slices and compare Antidote keys (atoms) int aKeySize = checkAndTrimFirstBytes(ac); From 8a9b7dc56b2a5ebc6dd0a0a384cb5635289e89af Mon Sep 17 00:00:00 2001 From: santialvarezcolombo Date: Sun, 17 Jan 2016 18:19:24 +0100 Subject: [PATCH 05/24] parse list size method --- c_src/antidote.cc | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/c_src/antidote.cc b/c_src/antidote.cc index 40e3f522..81ec0d6f 100644 --- a/c_src/antidote.cc +++ b/c_src/antidote.cc @@ -68,6 +68,23 @@ namespace leveldb { return (int) sc[0]; } + // Given a slice, checks that a list follows, and returns its size. + static int checkList(Slice &s) { + // LIST_EXT = 108 + assert(s[0] == (char) 108); + s.remove_prefix(1); + + // parse list size + unsigned char size[4]; + size[3] = s[0]; + size[2] = s[1]; + size[1] = s[2]; + size[0] = s[3]; + + s.remove_prefix(4); + return *(int *)size; + } + // No need to shorten keys since it's fixed size. virtual void FindShortestSeparator(std::string* start, const Slice& limit) const { From 56aa2ff5ca0e2586d049f5858b80e1aae7ba4121 Mon Sep 17 00:00:00 2001 From: santialvarezcolombo Date: Sun, 17 Jan 2016 18:51:27 +0100 Subject: [PATCH 06/24] key compare finished. started with VC comparison --- c_src/antidote.cc | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/c_src/antidote.cc b/c_src/antidote.cc index 81ec0d6f..2f3262a7 100644 --- a/c_src/antidote.cc +++ b/c_src/antidote.cc @@ -37,7 +37,19 @@ namespace leveldb { Slice aKey = Slice(ac.data(), aKeySize); Slice bKey = Slice(bc.data(), bKeySize); - return aKey.compare(bKey); + int key_compare = aKey.compare(bKey); + + if(key_compare) { + return key_compare; + } + + // If keys are equal, continue with the vector clock + // First trim the key + ac.remove_prefix(aKeySize); + bc.remove_prefix(bKeySize); + + //return 1 for now + return 1; } // Given a slice, checks that the first bytes match Erlang From e20ac192c98de73c1907c7f60d24bf64f1f144b1 Mon Sep 17 00:00:00 2001 From: santialvarezcolombo Date: Sun, 17 Jan 2016 19:33:29 +0100 Subject: [PATCH 07/24] first version for VC comparison --- c_src/antidote.cc | 70 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 59 insertions(+), 11 deletions(-) diff --git a/c_src/antidote.cc b/c_src/antidote.cc index 2f3262a7..2b9ced49 100644 --- a/c_src/antidote.cc +++ b/c_src/antidote.cc @@ -5,6 +5,7 @@ #include "port/port.h" #include "util/logging.h" #include +#include using namespace std; namespace leveldb { @@ -48,8 +49,12 @@ namespace leveldb { ac.remove_prefix(aKeySize); bc.remove_prefix(bKeySize); - //return 1 for now - return 1; + // Check lists + int aVCSize = checkList(ac); + int bVCSize = checkList(bc); + + // Parse and compare VCs + return compareVCs(parseVCMap(ac, aVCSize), parseVCMap(bc, bVCSize)); } // Given a slice, checks that the first bytes match Erlang @@ -84,17 +89,60 @@ namespace leveldb { static int checkList(Slice &s) { // LIST_EXT = 108 assert(s[0] == (char) 108); - s.remove_prefix(1); + return parseInt(s); + } + + static map parseVCMap(Slice &s, int size) { + map VC; + int key, value; + while(size > 0) { + key = parseInt(s); + value = parseInt(s); + VC[key] = value; + } + return VC; + } - // parse list size - unsigned char size[4]; - size[3] = s[0]; - size[2] = s[1]; - size[1] = s[2]; - size[0] = s[3]; + // Given a Slice parses a SMALL_INTEGER_EXT (97) or INTEGER_EXT (98) + static int parseInt(Slice &s) { + assert(s[0] == (char) 97 || s[0] == (char) 98); + int res; + if (s[0] == (char) 97) { + res = (int) s[1]; + s.remove_prefix(2); + } else { + unsigned char size[4]; + size[3] = s[1]; + size[2] = s[2]; + size[1] = s[3]; + size[0] = s[4]; + + s.remove_prefix(5); + res = *(int *) size; + } + return res; + } - s.remove_prefix(4); - return *(int *)size; + static int compareVCs(map a, map b) { + if (a.size() != b.size()) { + // TODO check what to do in this case + return 1; + } + map::iterator keyIt; + for(map::iterator iterator = a.begin(); + iterator != a.end(); iterator++) { + keyIt = b.find(iterator->first); + if(keyIt == b.end()) { // Key sets are != + // TODO check what to do in this case + return 1; + } + if(iterator->second > keyIt->second) { + continue; + } else { + return -1; + } + } + return 1; } // No need to shorten keys since it's fixed size. From dc7acf4ad3229510f8f1c7f46349e595937d39a4 Mon Sep 17 00:00:00 2001 From: santialvarezcolombo Date: Sun, 17 Jan 2016 19:40:51 +0100 Subject: [PATCH 08/24] revert change in checkList method --- c_src/antidote.cc | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/c_src/antidote.cc b/c_src/antidote.cc index 2b9ced49..3f3dae8b 100644 --- a/c_src/antidote.cc +++ b/c_src/antidote.cc @@ -89,7 +89,17 @@ namespace leveldb { static int checkList(Slice &s) { // LIST_EXT = 108 assert(s[0] == (char) 108); - return parseInt(s); + s.remove_prefix(1); + + // parse list size + unsigned char size[4]; + size[3] = s[0]; + size[2] = s[1]; + size[1] = s[2]; + size[0] = s[3]; + + s.remove_prefix(4); + return *(int *)size; } static map parseVCMap(Slice &s, int size) { From a6b95810e50f0a6bc5c5c6082332055007a1b715 Mon Sep 17 00:00:00 2001 From: santialvarezcolombo Date: Sun, 17 Jan 2016 19:46:52 +0100 Subject: [PATCH 09/24] added missing tuple parsing --- c_src/antidote.cc | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/c_src/antidote.cc b/c_src/antidote.cc index 3f3dae8b..1724ae2a 100644 --- a/c_src/antidote.cc +++ b/c_src/antidote.cc @@ -106,6 +106,7 @@ namespace leveldb { map VC; int key, value; while(size > 0) { + checkTwoElementTuple(s); key = parseInt(s); value = parseInt(s); VC[key] = value; @@ -113,6 +114,15 @@ namespace leveldb { return VC; } + static void checkTwoElementTuple(Slice &s) { + // SMALL_TUPLE_EXT == 104 + assert(s[0] == (char) 104); + s.remove_prefix(1); + // LENGTH == 2 (DC, clock) + assert(s[0] == (char) 2); + s.remove_prefix(1); + } + // Given a Slice parses a SMALL_INTEGER_EXT (97) or INTEGER_EXT (98) static int parseInt(Slice &s) { assert(s[0] == (char) 97 || s[0] == (char) 98); From 53df8127e85117a8510296bbab1de2cfc2f8afae Mon Sep 17 00:00:00 2001 From: santialvarezcolombo Date: Sun, 17 Jan 2016 19:59:22 +0100 Subject: [PATCH 10/24] revrse sorting order. most recent VCs first --- c_src/antidote.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/c_src/antidote.cc b/c_src/antidote.cc index 1724ae2a..a8207985 100644 --- a/c_src/antidote.cc +++ b/c_src/antidote.cc @@ -110,6 +110,7 @@ namespace leveldb { key = parseInt(s); value = parseInt(s); VC[key] = value; + size--; } return VC; } @@ -146,7 +147,7 @@ namespace leveldb { static int compareVCs(map a, map b) { if (a.size() != b.size()) { // TODO check what to do in this case - return 1; + return -1; } map::iterator keyIt; for(map::iterator iterator = a.begin(); @@ -154,15 +155,15 @@ namespace leveldb { keyIt = b.find(iterator->first); if(keyIt == b.end()) { // Key sets are != // TODO check what to do in this case - return 1; + return -1; } if(iterator->second > keyIt->second) { continue; } else { - return -1; + return 1; } } - return 1; + return -1; } // No need to shorten keys since it's fixed size. From 43c962cb27cb9305e830ce4f57ebe4b07117883c Mon Sep 17 00:00:00 2001 From: santialvarezcolombo Date: Thu, 21 Jan 2016 10:05:54 +0100 Subject: [PATCH 11/24] added code to provide folding --- c_src/antidote.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/c_src/antidote.cc b/c_src/antidote.cc index a8207985..722fb274 100644 --- a/c_src/antidote.cc +++ b/c_src/antidote.cc @@ -42,6 +42,12 @@ namespace leveldb { if(key_compare) { return key_compare; + } else { + // If we are supplied with a key that only contains + // the antidote key, we can't continue parsing, therefore + // return -1 or 1 according to which key is the shorter. + if ((ac.size() - aKeySize) == 0) return -1; + if ((bc.size() - bKeySize) == 0) return 1; } // If keys are equal, continue with the vector clock From 415e524ae9b3517078f99ab2678e46322876867f Mon Sep 17 00:00:00 2001 From: santialvarezcolombo Date: Thu, 21 Jan 2016 10:40:07 +0100 Subject: [PATCH 12/24] refactored VCs comparator to treat keys with different amount of VCs --- c_src/antidote.cc | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/c_src/antidote.cc b/c_src/antidote.cc index 722fb274..54110dbc 100644 --- a/c_src/antidote.cc +++ b/c_src/antidote.cc @@ -151,16 +151,13 @@ namespace leveldb { } static int compareVCs(map a, map b) { - if (a.size() != b.size()) { - // TODO check what to do in this case - return -1; - } map::iterator keyIt; - for(map::iterator iterator = a.begin(); - iterator != a.end(); iterator++) { - keyIt = b.find(iterator->first); - if(keyIt == b.end()) { // Key sets are != - // TODO check what to do in this case + for(map::iterator iterator = b.begin(); + iterator != b.end(); iterator++) { + keyIt = a.find(iterator->first); + if(keyIt == a.end()) { // Key sets are != + // we return -1 since a doesn't contain that key + // an therefore we should treat is as a 0. return -1; } if(iterator->second > keyIt->second) { From 7cde4b30a591d41fab7f7d96b117dbde9af70476 Mon Sep 17 00:00:00 2001 From: santialvarezcolombo Date: Thu, 21 Jan 2016 10:40:28 +0100 Subject: [PATCH 13/24] refactored VCs comparator to treat keys with different amount of VCs --- c_src/antidote.cc | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/c_src/antidote.cc b/c_src/antidote.cc index 54110dbc..986a8f6c 100644 --- a/c_src/antidote.cc +++ b/c_src/antidote.cc @@ -150,23 +150,33 @@ namespace leveldb { return res; } + // This method returns -1 * the comparison value, since + // we are sorting keys from oldest to newest first. static int compareVCs(map a, map b) { + if (a.size() > b.size()) { + // a is "newer" since it contains more keys. + return -1; + } + if (a.size() < b.size()) { + // b is "newer" since it contains more keys. + return 1; + } map::iterator keyIt; for(map::iterator iterator = b.begin(); iterator != b.end(); iterator++) { keyIt = a.find(iterator->first); if(keyIt == a.end()) { // Key sets are != - // we return -1 since a doesn't contain that key + // we return 1 since a doesn't contain that key // an therefore we should treat is as a 0. return -1; } if(iterator->second > keyIt->second) { continue; } else { - return 1; + return -1; } } - return -1; + return 1; } // No need to shorten keys since it's fixed size. From 89ca78725cd56e5254217e48c724b99b48ffdbf7 Mon Sep 17 00:00:00 2001 From: santialvarezcolombo Date: Thu, 21 Jan 2016 10:46:48 +0100 Subject: [PATCH 14/24] fixed bug while parsing ints --- c_src/antidote.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/c_src/antidote.cc b/c_src/antidote.cc index 986a8f6c..559fc175 100644 --- a/c_src/antidote.cc +++ b/c_src/antidote.cc @@ -135,7 +135,9 @@ namespace leveldb { assert(s[0] == (char) 97 || s[0] == (char) 98); int res; if (s[0] == (char) 97) { - res = (int) s[1]; + unsigned char size[1]; + size[0] = s[1]; + res = *(int *) size; s.remove_prefix(2); } else { unsigned char size[4]; From e466ab0c8a0a76027c7bc95b147b99edd18107cc Mon Sep 17 00:00:00 2001 From: santialvarezcolombo Date: Thu, 21 Jan 2016 11:15:30 +0100 Subject: [PATCH 15/24] VCs keys are now atoms instead of ints --- c_src/antidote.cc | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/c_src/antidote.cc b/c_src/antidote.cc index 559fc175..55818368 100644 --- a/c_src/antidote.cc +++ b/c_src/antidote.cc @@ -108,12 +108,13 @@ namespace leveldb { return *(int *)size; } - static map parseVCMap(Slice &s, int size) { - map VC; - int key, value; + static map parseVCMap(Slice &s, int size) { + map VC; + int value; + string key; while(size > 0) { checkTwoElementTuple(s); - key = parseInt(s); + key = parseAtom(s); value = parseInt(s); VC[key] = value; size--; @@ -152,9 +153,26 @@ namespace leveldb { return res; } + // Given a Slice parses an ATOM_EXT + static string parseAtom(Slice &s) { + // ATOM_EXT = 100 + assert(s[0] == (char) 100); + s.remove_prefix(1); + + // LENGTH + Slice sc = Slice(s.data(), 2); + s.remove_prefix(2); + sc.remove_prefix(1); + + // Create the result string and trim its sice from the Slice. + string res (s.data(), sc[0]); + s.remove_prefix(sc[0]); + return res; + } + // This method returns -1 * the comparison value, since // we are sorting keys from oldest to newest first. - static int compareVCs(map a, map b) { + static int compareVCs(map a, map b) { if (a.size() > b.size()) { // a is "newer" since it contains more keys. return -1; @@ -163,8 +181,8 @@ namespace leveldb { // b is "newer" since it contains more keys. return 1; } - map::iterator keyIt; - for(map::iterator iterator = b.begin(); + map::iterator keyIt; + for(map::iterator iterator = b.begin(); iterator != b.end(); iterator++) { keyIt = a.find(iterator->first); if(keyIt == a.end()) { // Key sets are != From a733095f1499cdf046c02df7a70c04235cc6a332 Mon Sep 17 00:00:00 2001 From: santialvarezcolombo Date: Sat, 23 Jan 2016 10:51:34 +0100 Subject: [PATCH 16/24] Parsing for empty lists (empty snapshots) --- c_src/antidote.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/c_src/antidote.cc b/c_src/antidote.cc index 55818368..8e32285d 100644 --- a/c_src/antidote.cc +++ b/c_src/antidote.cc @@ -93,6 +93,12 @@ namespace leveldb { // Given a slice, checks that a list follows, and returns its size. static int checkList(Slice &s) { + // NIL_EXT = 106 (Empty list = Empty Snapshot) + if(s[0] == (char) 106) { + s.remove_prefix(1); + return 0; + } + // LIST_EXT = 108 assert(s[0] == (char) 108); s.remove_prefix(1); From ac4942b4076450059237d1656d9f7fc4e7e6e7c9 Mon Sep 17 00:00:00 2001 From: santialvarezcolombo Date: Sat, 23 Jan 2016 13:31:15 +0100 Subject: [PATCH 17/24] fix bug while comparing keys with != number of DCs --- c_src/antidote.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/c_src/antidote.cc b/c_src/antidote.cc index 8e32285d..81bdcbdd 100644 --- a/c_src/antidote.cc +++ b/c_src/antidote.cc @@ -194,7 +194,7 @@ namespace leveldb { if(keyIt == a.end()) { // Key sets are != // we return 1 since a doesn't contain that key // an therefore we should treat is as a 0. - return -1; + return 1; } if(iterator->second > keyIt->second) { continue; From 59d1133802551ddb413b59b4d57b64b900277902 Mon Sep 17 00:00:00 2001 From: santialvarezcolombo Date: Sun, 24 Jan 2016 18:40:15 +0100 Subject: [PATCH 18/24] first implementation to use big numbers --- c_src/antidote.cc | 80 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 68 insertions(+), 12 deletions(-) diff --git a/c_src/antidote.cc b/c_src/antidote.cc index 81bdcbdd..ace08486 100644 --- a/c_src/antidote.cc +++ b/c_src/antidote.cc @@ -6,6 +6,7 @@ #include "util/logging.h" #include #include +#include using namespace std; namespace leveldb { @@ -114,9 +115,9 @@ namespace leveldb { return *(int *)size; } - static map parseVCMap(Slice &s, int size) { - map VC; - int value; + static map parseVCMap(Slice &s, int size) { + map VC; + unsigned long long int value; string key; while(size > 0) { checkTwoElementTuple(s); @@ -137,14 +138,25 @@ namespace leveldb { s.remove_prefix(1); } - // Given a Slice parses a SMALL_INTEGER_EXT (97) or INTEGER_EXT (98) - static int parseInt(Slice &s) { - assert(s[0] == (char) 97 || s[0] == (char) 98); - int res; + // Given a Slice parses a SMALL_INTEGER_EXT (97), INTEGER_EXT (98) + // SMALL_BIG_EXT (110) or LARGE_BIG_EXT (111) + static unsigned long long int parseInt(Slice &s) { + assert(s[0] == (char) 97 || s[0] == (char) 98 + || s[0] == (char) 110 || s[0] == (char) 111); + + if (s[0] == (char) 97 || s[0] == (char) 98) { + return parseSmallInt(s); + } + + return parseBigInt(s); + } + + static unsigned long long int parseSmallInt(Slice &s) { + unsigned long long int res; if (s[0] == (char) 97) { unsigned char size[1]; size[0] = s[1]; - res = *(int *) size; + res = *(unsigned long long int *) size; s.remove_prefix(2); } else { unsigned char size[4]; @@ -154,11 +166,54 @@ namespace leveldb { size[0] = s[4]; s.remove_prefix(5); - res = *(int *) size; + res = *(unsigned long long int *) size; } return res; } + static unsigned long long int parseBigInt(Slice &s) { + int intSize; + unsigned long long int res = 0; + if (s[0] == (char) 110) { + unsigned char size[1]; + size[0] = s[1]; + s.remove_prefix(2); + intSize = *(int *) size; + } else { + unsigned char size[4]; + size[3] = s[1]; + size[2] = s[2]; + size[1] = s[3]; + size[0] = s[4]; + + s.remove_prefix(5); + intSize = *(int *) size; + } + // Clock time can't be negative + assert(s[0] == (char) 48); + s.remove_prefix(1); + unsigned char current[1]; + int originalSize = intSize; + while (intSize > 0) { + res += ((*(int *) current) * power(256, originalSize - intSize)); + s.remove_prefix(1); + intSize--; + } + return res; + } + + static unsigned long long int power(unsigned long long int base, int exp) { + unsigned long long int result = 1ULL; + while(exp > 0) { + if (exp & 1) { + result *= (unsigned long long int) base; + } + exp >>= 1; + base *= base; + } + return result; + } + // Given a Slice parses an ATOM_EXT static string parseAtom(Slice &s) { // ATOM_EXT = 100 @@ -178,7 +233,8 @@ namespace leveldb { // This method returns -1 * the comparison value, since // we are sorting keys from oldest to newest first. - static int compareVCs(map a, map b) { + static int compareVCs(map a, + map b) { if (a.size() > b.size()) { // a is "newer" since it contains more keys. return -1; @@ -187,8 +243,8 @@ namespace leveldb { // b is "newer" since it contains more keys. return 1; } - map::iterator keyIt; - for(map::iterator iterator = b.begin(); + map::iterator keyIt; + for(map::iterator iterator = b.begin(); iterator != b.end(); iterator++) { keyIt = a.find(iterator->first); if(keyIt == a.end()) { // Key sets are != From 3b62c91646e0696836bd80ac82a71dbaadf68771 Mon Sep 17 00:00:00 2001 From: santialvarezcolombo Date: Sun, 24 Jan 2016 18:53:03 +0100 Subject: [PATCH 19/24] fixed assert --- c_src/antidote.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/c_src/antidote.cc b/c_src/antidote.cc index ace08486..2f97dcbf 100644 --- a/c_src/antidote.cc +++ b/c_src/antidote.cc @@ -189,8 +189,8 @@ namespace leveldb { s.remove_prefix(5); intSize = *(int *) size; } - // Clock time can't be negative - assert(s[0] == (char) 48); + // Clock time can't be negative, therfore this byte must be 0 + assert((int) s[0] == 0); s.remove_prefix(1); unsigned char current[1]; int originalSize = intSize; From 539552ec2779fdbc36903d75e103e2e7a2292f7b Mon Sep 17 00:00:00 2001 From: santialvarezcolombo Date: Sun, 24 Jan 2016 19:17:45 +0100 Subject: [PATCH 20/24] fixed power --- c_src/antidote.cc | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/c_src/antidote.cc b/c_src/antidote.cc index 2f97dcbf..dcdd712d 100644 --- a/c_src/antidote.cc +++ b/c_src/antidote.cc @@ -189,12 +189,13 @@ namespace leveldb { s.remove_prefix(5); intSize = *(int *) size; } - // Clock time can't be negative, therfore this byte must be 0 + // Clock time can't be negative, therfore this byte must be 0 assert((int) s[0] == 0); s.remove_prefix(1); unsigned char current[1]; int originalSize = intSize; while (intSize > 0) { + current[0] = s[0]; res += ((*(int *) current) * power(256, originalSize - intSize)); s.remove_prefix(1); intSize--; @@ -203,13 +204,10 @@ namespace leveldb { } static unsigned long long int power(unsigned long long int base, int exp) { - unsigned long long int result = 1ULL; + unsigned long long int result = 1; while(exp > 0) { - if (exp & 1) { - result *= (unsigned long long int) base; - } - exp >>= 1; - base *= base; + result *= base; + exp--; } return result; } From 8b54842268e9287cf23284e63465de9fdcaf6242 Mon Sep 17 00:00:00 2001 From: santialvarezcolombo Date: Mon, 28 Mar 2016 16:46:25 -0400 Subject: [PATCH 21/24] New comparator method, asuming VCs is a list sorted by DCs --- c_src/antidote.cc | 94 ++++++++++++++++++++++++++++------------------- 1 file changed, 56 insertions(+), 38 deletions(-) diff --git a/c_src/antidote.cc b/c_src/antidote.cc index dcdd712d..dd6c3e16 100644 --- a/c_src/antidote.cc +++ b/c_src/antidote.cc @@ -30,6 +30,10 @@ namespace leveldb { return 0; } + if ((a[3] != (char) 100) || (b[3] != (char) 100)) { + return 1; + } + Slice ac = Slice(a.data(), a.size()), bc = Slice(b.data(), b.size()); // Trim Slices and compare Antidote keys (atoms) @@ -61,7 +65,7 @@ namespace leveldb { int bVCSize = checkList(bc); // Parse and compare VCs - return compareVCs(parseVCMap(ac, aVCSize), parseVCMap(bc, bVCSize)); + return compareVCs(ac, aVCSize, bc, bVCSize); } // Given a slice, checks that the first bytes match Erlang @@ -115,20 +119,6 @@ namespace leveldb { return *(int *)size; } - static map parseVCMap(Slice &s, int size) { - map VC; - unsigned long long int value; - string key; - while(size > 0) { - checkTwoElementTuple(s); - key = parseAtom(s); - value = parseInt(s); - VC[key] = value; - size--; - } - return VC; - } - static void checkTwoElementTuple(Slice &s) { // SMALL_TUPLE_EXT == 104 assert(s[0] == (char) 104); @@ -189,7 +179,7 @@ namespace leveldb { s.remove_prefix(5); intSize = *(int *) size; } - // Clock time can't be negative, therfore this byte must be 0 + // Clock time can't be negative, therefore this byte must be 0 assert((int) s[0] == 0); s.remove_prefix(1); unsigned char current[1]; @@ -223,7 +213,7 @@ namespace leveldb { s.remove_prefix(2); sc.remove_prefix(1); - // Create the result string and trim its sice from the Slice. + // Create the result string and trim its size from the Slice. string res (s.data(), sc[0]); s.remove_prefix(sc[0]); return res; @@ -231,32 +221,60 @@ namespace leveldb { // This method returns -1 * the comparison value, since // we are sorting keys from oldest to newest first. - static int compareVCs(map a, - map b) { - if (a.size() > b.size()) { - // a is "newer" since it contains more keys. - return -1; - } - if (a.size() < b.size()) { - // b is "newer" since it contains more keys. - return 1; + static int compareVCs(Slice a, int aVCSize, Slice b, int bVCSize) { + // If any of them is an empty snapshot, + // return the comparison of the sizes + if (aVCSize == 0 || bVCSize == 0) { + return sizeComparison(aVCSize, bVCSize); } - map::iterator keyIt; - for(map::iterator iterator = b.begin(); - iterator != b.end(); iterator++) { - keyIt = a.find(iterator->first); - if(keyIt == a.end()) { // Key sets are != - // we return 1 since a doesn't contain that key - // an therefore we should treat is as a 0. - return 1; - } - if(iterator->second > keyIt->second) { - continue; + unsigned long long int valueA, valueB; + string keyA, keyB; + int aSize = aVCSize, bSize = bVCSize; + // Iterate the vector clocks and compare them + while(aSize > 0 && bSize > 0) { + checkTwoElementTuple(a); + checkTwoElementTuple(b); + keyA = parseAtom(a); + keyB = parseAtom(b); + valueA = parseInt(a); + valueB = parseInt(b); + + // Keys are sorted, so we leverage that + if(keyA.compare(keyB) == 0) { + // Same key + if(valueB == valueA) { + // Values for this key are equal, + // continue to next key + aSize--; + bSize--; + continue; + } + // Values are different, so return the comparison + if(valueB > valueA) { + return 1; + } else { + return -1; + } } else { + // Key is different so return the comparison + return keyA.compare(keyB); + } + } + // VCs are the same until now + // Check if any of them has more keys + return sizeComparison(aSize, bSize); + } + + static int sizeComparison(int sizeA, int sizeB) { + if (sizeA == 0 && sizeB == 0){ + return 0; + } else { + if (sizeA > 0 && sizeB == 0) { return -1; + } else { + return 1; } } - return 1; } // No need to shorten keys since it's fixed size. From ad1c770906c8df5e92b3054a42034434e122ec87 Mon Sep 17 00:00:00 2001 From: santialvarezcolombo Date: Mon, 26 Sep 2016 18:54:26 -0300 Subject: [PATCH 22/24] Fixed comparison of keys with same VC to deferentiate op vs snaps. --- c_src/antidote.cc | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/c_src/antidote.cc b/c_src/antidote.cc index dd6c3e16..53530a6f 100644 --- a/c_src/antidote.cc +++ b/c_src/antidote.cc @@ -262,7 +262,14 @@ namespace leveldb { } // VCs are the same until now // Check if any of them has more keys - return sizeComparison(aSize, bSize); + int vcSize = sizeComparison(aSize, bSize); + if (vcSize != 0) { + return vcSize; + } else { + // If the VC is equal, check if it's an op or a snap. + // If this returns 0, the key is identical + return sizeComparison(a.size(), b.size()); + } } static int sizeComparison(int sizeA, int sizeB) { From acfd2bd10511e43fa52eea7b7fe88fc32bbb0ec5 Mon Sep 17 00:00:00 2001 From: santialvarezcolombo Date: Thu, 20 Oct 2016 19:24:53 -0300 Subject: [PATCH 23/24] New comparator ignoring VCs name and only taking into consideration the values of each one from max to min --- c_src/antidote.cc | 43 ++++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/c_src/antidote.cc b/c_src/antidote.cc index 53530a6f..41aec901 100644 --- a/c_src/antidote.cc +++ b/c_src/antidote.cc @@ -239,25 +239,18 @@ namespace leveldb { valueA = parseInt(a); valueB = parseInt(b); - // Keys are sorted, so we leverage that - if(keyA.compare(keyB) == 0) { - // Same key - if(valueB == valueA) { - // Values for this key are equal, - // continue to next key - aSize--; - bSize--; - continue; - } - // Values are different, so return the comparison - if(valueB > valueA) { - return 1; - } else { - return -1; - } + if(valueB == valueA) { + // Values for this key are equal, + // continue to next key + aSize--; + bSize--; + continue; + } + // Values are different, so return the comparison + if(valueB > valueA) { + return 1; } else { - // Key is different so return the comparison - return keyA.compare(keyB); + return -1; } } // VCs are the same until now @@ -265,10 +258,18 @@ namespace leveldb { int vcSize = sizeComparison(aSize, bSize); if (vcSize != 0) { return vcSize; + } + + // If VCs are equal, compare Hashes and op/snap + // Usint the Slice compare method + return -1 * a.compare(b); + } + + static int hashComparison(int hashA, int hashB) { + if(hashB > hashA) { + return 1; } else { - // If the VC is equal, check if it's an op or a snap. - // If this returns 0, the key is identical - return sizeComparison(a.size(), b.size()); + return -1; } } From e6d633e96a237b86fa02f2c9c2d2c855c6f27499 Mon Sep 17 00:00:00 2001 From: santialvarezcolombo Date: Sun, 23 Oct 2016 13:19:45 -0300 Subject: [PATCH 24/24] New comparator function using only the MAX value in the VC --- c_src/antidote.cc | 141 +++++++--------------------------------------- 1 file changed, 20 insertions(+), 121 deletions(-) diff --git a/c_src/antidote.cc b/c_src/antidote.cc index 41aec901..97861d41 100644 --- a/c_src/antidote.cc +++ b/c_src/antidote.cc @@ -55,17 +55,31 @@ namespace leveldb { if ((bc.size() - bKeySize) == 0) return 1; } - // If keys are equal, continue with the vector clock + // If keys are equal, continue with the max value // First trim the key ac.remove_prefix(aKeySize); bc.remove_prefix(bKeySize); - // Check lists - int aVCSize = checkList(ac); - int bVCSize = checkList(bc); + unsigned long long int valueA, valueB; + valueA = parseInt(ac); + valueB = parseInt(bc); + + if(valueA == valueB) { + // If we are supplied with a key that only contains + // the antidote key, and max value we can't continue parsing, + // therefore return -1 or 1 according to which key is the shorter. + if(ac.size() == 0) return -1; + if(bc.size() == 0) return 1; + // Keys are full, so return the comparison of the hash, etc.. + return -1 * ac.compare(bc); + } - // Parse and compare VCs - return compareVCs(ac, aVCSize, bc, bVCSize); + // Max values are != so return their comparison + if(valueB > valueA) { + return 1; + } else { + return -1; + } } // Given a slice, checks that the first bytes match Erlang @@ -96,38 +110,6 @@ namespace leveldb { return (int) sc[0]; } - // Given a slice, checks that a list follows, and returns its size. - static int checkList(Slice &s) { - // NIL_EXT = 106 (Empty list = Empty Snapshot) - if(s[0] == (char) 106) { - s.remove_prefix(1); - return 0; - } - - // LIST_EXT = 108 - assert(s[0] == (char) 108); - s.remove_prefix(1); - - // parse list size - unsigned char size[4]; - size[3] = s[0]; - size[2] = s[1]; - size[1] = s[2]; - size[0] = s[3]; - - s.remove_prefix(4); - return *(int *)size; - } - - static void checkTwoElementTuple(Slice &s) { - // SMALL_TUPLE_EXT == 104 - assert(s[0] == (char) 104); - s.remove_prefix(1); - // LENGTH == 2 (DC, clock) - assert(s[0] == (char) 2); - s.remove_prefix(1); - } - // Given a Slice parses a SMALL_INTEGER_EXT (97), INTEGER_EXT (98) // SMALL_BIG_EXT (110) or LARGE_BIG_EXT (111) static unsigned long long int parseInt(Slice &s) { @@ -202,89 +184,6 @@ namespace leveldb { return result; } - // Given a Slice parses an ATOM_EXT - static string parseAtom(Slice &s) { - // ATOM_EXT = 100 - assert(s[0] == (char) 100); - s.remove_prefix(1); - - // LENGTH - Slice sc = Slice(s.data(), 2); - s.remove_prefix(2); - sc.remove_prefix(1); - - // Create the result string and trim its size from the Slice. - string res (s.data(), sc[0]); - s.remove_prefix(sc[0]); - return res; - } - - // This method returns -1 * the comparison value, since - // we are sorting keys from oldest to newest first. - static int compareVCs(Slice a, int aVCSize, Slice b, int bVCSize) { - // If any of them is an empty snapshot, - // return the comparison of the sizes - if (aVCSize == 0 || bVCSize == 0) { - return sizeComparison(aVCSize, bVCSize); - } - unsigned long long int valueA, valueB; - string keyA, keyB; - int aSize = aVCSize, bSize = bVCSize; - // Iterate the vector clocks and compare them - while(aSize > 0 && bSize > 0) { - checkTwoElementTuple(a); - checkTwoElementTuple(b); - keyA = parseAtom(a); - keyB = parseAtom(b); - valueA = parseInt(a); - valueB = parseInt(b); - - if(valueB == valueA) { - // Values for this key are equal, - // continue to next key - aSize--; - bSize--; - continue; - } - // Values are different, so return the comparison - if(valueB > valueA) { - return 1; - } else { - return -1; - } - } - // VCs are the same until now - // Check if any of them has more keys - int vcSize = sizeComparison(aSize, bSize); - if (vcSize != 0) { - return vcSize; - } - - // If VCs are equal, compare Hashes and op/snap - // Usint the Slice compare method - return -1 * a.compare(b); - } - - static int hashComparison(int hashA, int hashB) { - if(hashB > hashA) { - return 1; - } else { - return -1; - } - } - - static int sizeComparison(int sizeA, int sizeB) { - if (sizeA == 0 && sizeB == 0){ - return 0; - } else { - if (sizeA > 0 && sizeB == 0) { - return -1; - } else { - return 1; - } - } - } - // No need to shorten keys since it's fixed size. virtual void FindShortestSeparator(std::string* start, const Slice& limit) const {