Merge pull request #16 from resilientdb/client_memory

Client Memory #11 and config file bug fixed

Makefile

@@ -8,8 +8,8 @@ SQLITE=deps/sqlite-autoconf-3290000/build
 
 .SUFFIXES: .o .cpp .h
 
-SRC_DIRS = ./ ./benchmarks/ ./client/ ./transport/ ./system/ ./statistics/ ./blockchain/ ./db/ ./smart_contracts/
-DEPS = -I. -I./benchmarks -I./client/ -I./transport -I./system -I./statistics -I./blockchain -I./smart_contracts -I$(JEMALLOC)/include -I$(NNMSG)/include -I$(BOOST) -I$(CRYPTOPP) -I./db -I$(SQLITE)/include
+SRC_DIRS = ./ ./benchmarks/ ./client/ ./transport/ ./system/ ./statistics/ ./blockchain/ ./db/ ./smart_contracts/ ./data_structures
+DEPS = -I. -I./benchmarks -I./client/ -I./transport -I./system -I./statistics -I./blockchain -I./smart_contracts  -I./data_structures -I$(JEMALLOC)/include -I$(NNMSG)/include -I$(BOOST) -I$(CRYPTOPP) -I./db -I$(SQLITE)/include
 
 CFLAGS += $(DEPS) -D NOGRAPHITE=1 -Werror -Wno-sizeof-pointer-memaccess
 LDFLAGS = -Wall -L. -L$(NNMSG)/lib -L$(JEMALLOC)/lib -Wl,-rpath,$(JEMALLOC)/lib -pthread -gdwarf-3 -lrt -std=c++0x -L$(CRYPTOPP) -L$(SQLITE)/lib

client/client_main.cpp

@@ -163,18 +163,6 @@ int main(int argc, char *argv[])
 
     printf("Done\n");
 
-#if !RBFT_ON
-    Message *clrsp = Message::create_message(CL_RSP);
-    for (uint64_t i = 0; i < 2 * g_client_node_cnt * g_inflight_max; i++)
-    {
-        for (uint64_t j = 0; j < g_node_cnt; j++)
-        {
-            ((ClientResponseMessage *)clrsp)->txn_id = UINT64_MAX;
-            clrspStore[i][j] = *((ClientResponseMessage *)clrsp);
-        }
-    }
-#endif
-
     // 2. spawn multiple threads
     uint64_t thd_cnt = g_client_thread_cnt;
     uint64_t cthd_cnt = thd_cnt;

data_structures/hash_map.h

@@ -0,0 +1,88 @@
+#ifndef _HASH_MAP_H_
+#define _HASH_MAPS_H_
+
+#include <iostream>
+#include <unordered_map>
+#include <mutex>
+
+template <class _key, class _val>
+class SpinLockMap
+{
+public:
+    SpinLockMap(){};
+    ~SpinLockMap() { this->map.clear(); }
+
+    bool exists(_key key)
+    {
+        bool result = false;
+        this->lock.lock();
+        result = this->map.count(key);
+        this->lock.unlock();
+        return result;
+    }
+    bool check_and_get(_key key, _val &val)
+    {
+        bool result = false;
+        this->lock.lock();
+        result = this->map.count(key);
+        if (result)
+            val = this->map[key];
+        this->lock.unlock();
+        return result;
+    }
+    _val get(_key key)
+    {
+        this->lock.lock();
+
+        assert(this->map.count(key));
+        _val temp = this->map[key];
+
+        this->lock.unlock();
+        return temp;
+    };
+
+    void add(_key key, _val value)
+    {
+        this->lock.lock();
+        this->map[key] = value;
+        this->lock.unlock();
+    };
+    int remove(_key key)
+    {
+
+        this->lock.lock();
+
+        int result = this->map.erase(key);
+
+        this->lock.unlock();
+        return result;
+    }
+
+    _val pop(_key key)
+    {
+        _val temp;
+
+        this->lock.lock();
+        if (this->map.count(key))
+        {
+            temp = this->map[key];
+            this->map.erase(key);
+        }
+        else
+        {
+            temp = 0;
+        }
+        this->lock.unlock();
+        return temp;
+    }
+    int size()
+    {
+        return map.size();
+    }
+
+private:
+    std::unordered_map<_key, _val> map;
+    std::mutex lock;
+};
+
+#endif

data_structures/hash_set.h

@@ -0,0 +1,58 @@
+#ifndef _HASH_MAP_H_
+#define _HASH_MAPS_H_
+
+#include <iostream>
+#include <set>
+#include <mutex>
+
+template <class _type>
+class SpinLockSet
+{
+public:
+    SpinLockSet(){};
+    ~SpinLockSet() { this->set.clear(); }
+
+    int exists(_type key)
+    {
+        bool result = false;
+        this->lock.lock();
+        result = this->set.count(key);
+        this->lock.unlock();
+        return result;
+    }
+    void add(_type key)
+    {
+        this->lock.lock();
+        this->set.insert(key);
+        this->lock.unlock();
+    };
+    int check_and_add(_type key)
+    {
+        bool result = false;
+        this->lock.lock();
+        result = this->set.count(key);
+        this->set.insert(key);
+        this->lock.unlock();
+        return result;
+    }
+    int remove(_type key)
+    {
+        this->lock.lock();
+
+        int result = this->set.erase(key);
+
+        this->lock.unlock();
+        return result;
+    }
+
+    int size()
+    {
+        return set.size();
+    }
+
+private:
+    std::set<_type> set;
+    std::mutex lock;
+};
+
+#endif

data_structures/shared_int.h

@@ -0,0 +1,47 @@
+#ifndef _HASH_MAP_H_
+#define _HASH_MAPS_H_
+
+#include <iostream>
+#include <unordered_map>
+#include <mutex>
+
+class SharedInt
+{
+public:
+    SharedInt(){};
+    ~SharedInt() {}
+
+    uint64_t get()
+    {
+        uint64_t result;
+        this->lock.lock();
+        result = this->value;
+        this->lock.unlock();
+        return result;
+    };
+    void set(uint64_t val)
+    {
+        this->lock.lock();
+        this->value = val;
+        this->lock.unlock();
+    };
+
+    void add(uint64_t val)
+    {
+        this->lock.lock();
+        this->value += val;
+        this->lock.unlock();
+    };
+    void sub(uint64_t val)
+    {
+        this->lock.lock();
+        this->value -= val;
+        this->lock.unlock();
+    }
+
+private:
+    uint64_t value;
+    std::mutex lock;
+};
+
+#endif

scripts/make_config.sh

@@ -39,6 +39,7 @@ echo -e "#define TIME_PROF_ENABLE false " >>config.h
 echo -e "#define FIN_BY_TIME true " >>config.h
 echo -e "// Number of transactions each client should send without waiting. " >>config.h
 echo -e "#define MAX_TXN_IN_FLIGHT $max_inf " >>config.h
+echo -e "#define MESSAGE_PER_BUFFER 1 " >>config.h
 echo -e "#define SERVER_GENERATE_QUERIES false  " >>config.h
 echo -e "#define MEM_ALLIGN 8 " >>config.h
 echo -e "#define THREAD_ALLOC false " >>config.h
@@ -220,7 +221,7 @@ echo -e "#define M100 1	// 100KB. " >>config.h
 echo -e "#define M200 2	// 200KB. " >>config.h
 echo -e "#define M400 3	// 400KB. " >>config.h
 echo -e "" >>config.h
-echo -e "#define EXT_DB SQL" >> config.h
+echo -e "#define EXT_DB MEMORY" >> config.h
 echo -e "#define MEMORY 1" >> config.h
 echo -e "#define SQL 2" >> config.h
 echo -e "#define SQL_PERSISTENT 3" >> config.h

scripts/startResilientDB.sh

@@ -4,82 +4,69 @@
 # This script allows to compile and run the code. You need to specify IP addresses of your servers and clients. The scripts expect three arguments and the result is stored in a folder named "results". Do create a folder with the name "results" before running this script.
 ######
 
-
-
-i=$1		# Argument 1 to script --> Number of replicas
-cli=$2		# Argument 2 to script --> Number of clients
-bsize=$3	# Argumnet 3 to script --> Batch Size
-
+i=$1     # Argument 1 to script --> Number of replicas
+cli=$2   # Argument 2 to script --> Number of clients
+bsize=$3 # Argumnet 3 to script --> Batch Size
 
 SNODES=(
-# Specify private IP addresses of your replicas (as below).
-#"10.128.0.246"
+	# Specify private IP addresses of your replicas (as below).
+	#"10.128.0.246"
 )
 
 CNODES=(
-# Specify Private IPs of your clients. 
-#"10.128.1.6"
+	# Specify Private IPs of your clients.
+	#"10.128.1.6"
 )
 
-
 rm ifconfig.txt hostnames.py
 
 # Building file ifconfig.txt
 #
 count=0
-while (($count < $i))
-do
-	echo ${SNODES[$count]} >> ifconfig.txt
-	count=$((count+1))
+while (($count < $i)); do
+	echo ${SNODES[$count]} >>ifconfig.txt
+	count=$((count + 1))
 done
 
 count=0
-while (($count < $cli))
-do
-	echo ${CNODES[$count]} >> ifconfig.txt
-	count=$((count+1))
+while (($count < $cli)); do
+	echo ${CNODES[$count]} >>ifconfig.txt
+	count=$((count + 1))
 done
 
-
 # Building file hostnames
 #
-echo "hostip = [" >> hostnames.py
+echo "hostip = [" >>hostnames.py
 count=0
-while (($count < $i))
-do
-	echo -e "\""${SNODES[$count]}"\"," >> hostnames.py
-	count=$((count+1))
+while (($count < $i)); do
+	echo -e "\""${SNODES[$count]}"\"," >>hostnames.py
+	count=$((count + 1))
 done
 
 count=0
-while (($count < $cli))
-do
-	echo -e "\""${CNODES[$count]}"\"," >> hostnames.py
-	count=$((count+1))
+while (($count < $cli)); do
+	echo -e "\""${CNODES[$count]}"\"," >>hostnames.py
+	count=$((count + 1))
 done
-echo "]" >> hostnames.py
-
+echo "]" >>hostnames.py
 
-
-echo "hostmach = [" >> hostnames.py
+echo "hostmach = [" >>hostnames.py
 count=0
-while (($count < $i))
-do
-	echo "\""${SNODES[$count]}"\"," >> hostnames.py
-	count=$((count+1))
+while (($count < $i)); do
+	echo "\""${SNODES[$count]}"\"," >>hostnames.py
+	count=$((count + 1))
 done
 
 count=0
-while (($count < $cli))
-do
-	echo "\""${CNODES[$count]}"\"," >> hostnames.py
-	count=$((count+1))
+while (($count < $cli)); do
+	echo "\""${CNODES[$count]}"\"," >>hostnames.py
+	count=$((count + 1))
 done
-echo "]" >> hostnames.py
-
+echo "]" >>hostnames.py
 
 # Compiling the Code
-make clean; make -j8
+make clean
+make -j8
 
 tm=0
 
@@ -91,11 +78,10 @@ cp hostnames.py scripts/
 cd scripts
 
 # Number of times you want to run the code (default 1)
-while [ $tm -lt 1 ]
-do
+while [ $tm -lt 1 ]; do
 	python3 simRun.py $i s${i}_c${cli}_results_PBFT_b${bsize}_run${tm}_node $tm
 
-	tm=$((tm+1))
+	tm=$((tm + 1))
 done
 
 # Go back

system/global.cpp

@@ -353,6 +353,10 @@ double idle_worker_times[THREAD_CNT] = {0};
 // Statistics to print output_thread_idle_times.
 double output_thd_idle_time[SEND_THREAD_CNT] = {0};
 
+// Maps for client response couting
+SpinLockMap<uint64_t, uint64_t> client_responses_count;
+SpinLockMap<uint64_t, ClientResponseMessage *> client_responses_directory;
+
 // Payload for messages.
 #if PAYLOAD_ENABLE
 #if PAYLOAD == M100