Contract map storage & Scilla IPC

cmake/InstallJsonrpccpp.cmake

@@ -23,8 +23,8 @@ set(CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>
                -DCMAKE_POSITION_INDEPENDENT_CODE=On
                -DBUILD_STATIC_LIBS=On
                -DBUILD_SHARED_LIBS=Off
-               -DUNIX_DOMAIN_SOCKET_SERVER=Off
-               -DUNIX_DOMAIN_SOCKET_CLIENT=Off
+               -DUNIX_DOMAIN_SOCKET_SERVER=On
+               -DUNIX_DOMAIN_SOCKET_CLIENT=On
                -DHTTP_SERVER=On
                -DHTTP_CLIENT=On
                -DCOMPILE_TESTS=Off

constants.xml

@@ -146,6 +146,7 @@
         <STATUS_RPC_PORT>4301</STATUS_RPC_PORT>
         <IP_TO_BIND>127.0.0.1</IP_TO_BIND>
         <ENABLE_STATUS_RPC>true</ENABLE_STATUS_RPC>
+        <SCILLA_IPC_SOCKET_PATH>/tmp/scillaipcsocket</SCILLA_IPC_SOCKET_PATH>
     </jsonrpc>
     <network_composition>
         <!-- Shard size will be automatically calculated if COMM_SIZE = 0 -->
@@ -223,6 +224,7 @@
         <REJOIN_NODE_NOT_IN_NETWORK>true</REJOIN_NODE_NOT_IN_NETWORK>
         <RESUME_BLACKLIST_DELAY_IN_SECONDS>30</RESUME_BLACKLIST_DELAY_IN_SECONDS>
         <INCRDB_DSNUMS_WITH_STATEDELTAS>10</INCRDB_DSNUMS_WITH_STATEDELTAS>
+        <CONTRACT_STATES_MIGRATED>false</CONTRACT_STATES_MIGRATED>
     </recovery>
     <smart_contract>
         <ENABLE_SC>false</ENABLE_SC>
@@ -239,6 +241,8 @@
         <OUTPUT_JSON>output.json</OUTPUT_JSON>
         <INPUT_CODE>input.scilla</INPUT_CODE>
         <ENABLE_SCILLA_MULTI_VERSION>true</ENABLE_SCILLA_MULTI_VERSION>
+        <FIELDS_MAP_DEPTH_INDICATOR>_fields_map_depth</FIELDS_MAP_DEPTH_INDICATOR>
+        <LOG_SC>true</LOG_SC>
     </smart_contract>
     <tests>
         <ENABLE_CHECK_PERFORMANCE_LOG>false</ENABLE_CHECK_PERFORMANCE_LOG>

constants_local.xml

@@ -145,6 +145,7 @@
         <STATUS_RPC_PORT>4301</STATUS_RPC_PORT>
         <IP_TO_BIND>127.0.0.1</IP_TO_BIND>
         <ENABLE_STATUS_RPC>true</ENABLE_STATUS_RPC>
+        <SCILLA_IPC_SOCKET_PATH>/tmp/scillaipcsocket</SCILLA_IPC_SOCKET_PATH>
     </jsonrpc>
     <network_composition>
         <!-- Shard size will be automatically calculated if COMM_SIZE = 0 -->
@@ -222,6 +223,7 @@
         <REJOIN_NODE_NOT_IN_NETWORK>true</REJOIN_NODE_NOT_IN_NETWORK>
         <RESUME_BLACKLIST_DELAY_IN_SECONDS>30</RESUME_BLACKLIST_DELAY_IN_SECONDS>
         <INCRDB_DSNUMS_WITH_STATEDELTAS>10</INCRDB_DSNUMS_WITH_STATEDELTAS>
+        <CONTRACT_STATES_MIGRATED>false</CONTRACT_STATES_MIGRATED>
     </recovery>
     <smart_contract>
         <ENABLE_SC>false</ENABLE_SC>
@@ -238,6 +240,8 @@
         <OUTPUT_JSON>output.json</OUTPUT_JSON>
         <INPUT_CODE>input.scilla</INPUT_CODE>
         <ENABLE_SCILLA_MULTI_VERSION>true</ENABLE_SCILLA_MULTI_VERSION>
+        <FIELDS_MAP_DEPTH_INDICATOR>_fields_map_depth</FIELDS_MAP_DEPTH_INDICATOR>
+        <LOG_SC>true</LOG_SC>
     </smart_contract>
     <tests>
         <ENABLE_CHECK_PERFORMANCE_LOG>false</ENABLE_CHECK_PERFORMANCE_LOG>

docker/dev/Dockerfile

@@ -40,13 +40,11 @@ RUN apt-get update \
     libleveldb-dev \
     libmicrohttpd-dev \
     libminiupnpc-dev \
-    libprotobuf-dev \
     libsnappy-dev \
     libssl-dev \
     libtool \
     ocl-icd-opencl-dev \
     pkg-config \
-    protobuf-compiler \
     python \
     python-pip \
     libsecp256k1-dev \

src/cmd/CMakeLists.txt

@@ -82,11 +82,19 @@ add_custom_command(TARGET zilliqa
         POST_BUILD
         COMMAND ${CMAKE_COMMAND} -E copy $<TARGET_FILE:restore> ${CMAKE_BINARY_DIR}/tests/Zilliqa)
 target_include_directories(restore PUBLIC ${CMAKE_SOURCE_DIR}/src)
+
 target_link_libraries(restore PUBLIC Node Mediator Validator)
 
 add_executable(genTxnBodiesFromS3 genTxnBodiesFromS3.cpp)
 add_custom_command(TARGET zilliqa
         POST_BUILD
         COMMAND ${CMAKE_COMMAND} -E copy $<TARGET_FILE:genTxnBodiesFromS3> ${CMAKE_BINARY_DIR}/tests/Zilliqa)
 target_include_directories(genTxnBodiesFromS3 PUBLIC ${CMAKE_SOURCE_DIR}/src)
-target_link_libraries(genTxnBodiesFromS3 PUBLIC AccountData Utils Persistence Server)
+target_link_libraries(genTxnBodiesFromS3 PUBLIC AccountData Utils Persistence Server)
+
+add_executable(data_migrate data_migrate.cpp)
+add_custom_command(TARGET zilliqa
+        POST_BUILD
+        COMMAND ${CMAKE_COMMAND} -E copy $<TARGET_FILE:data_migrate> ${CMAKE_BINARY_DIR}/tests/Zilliqa)
+target_include_directories(data_migrate PUBLIC ${CMAKE_SOURCE_DIR}/src)
+target_link_libraries(data_migrate PUBLIC Mediator Utils Persistence AccountData)

src/cmd/data_migrate.cpp

@@ -0,0 +1,54 @@
+/*
+ * Copyright (C) 2019 Zilliqa
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#include "libData/AccountData/AccountStore.h"
+#include "libMediator/Mediator.h"
+#include "libPersistence/BlockStorage.h"
+#include "libPersistence/Retriever.h"
+#include "libUtils/DataConversion.h"
+#include "libUtils/JsonUtils.h"
+
+/// Should be run from a folder named "persistence" consisting of the
+/// persistence
+
+using namespace std;
+int main() {
+  PairOfKey key;  // Dummy to initate mediator
+  Peer peer;
+
+  LOG_GENERAL(INFO, "Begin");
+
+  Mediator mediator(key, peer);
+  Retriever retriever(mediator);
+
+  LOG_GENERAL(INFO, "Start Retrieving States");
+
+  if (!retriever.RetrieveStates()) {
+    LOG_GENERAL(FATAL, "RetrieveStates failed");
+    return 0;
+  }
+
+  LOG_GENERAL(INFO, "finished RetrieveStates");
+
+  if (!retriever.MigrateContractStates()) {
+    LOG_GENERAL(WARNING, "MigrateContractStates failed");
+  } else {
+    LOG_GENERAL(INFO, "Migrate contract data finished");
+  }
+
+  return 0;
+}

src/common/Constants.cpp

@@ -15,6 +15,7 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 #include "Constants.h"
+#include "libUtils/DataConversion.h"
 #include "libUtils/SafeMath.h"
 
 #include <boost/property_tree/ptree.hpp>
@@ -301,6 +302,8 @@ const bool ENABLE_STATUS_RPC{
     ReadConstantString("ENABLE_STATUS_RPC", "node.jsonrpc.") == "true"};
 const unsigned int NUM_SHARD_PEER_TO_REVEAL{
     ReadConstantNumeric("NUM_SHARD_PEER_TO_REVEAL", "node.jsonrpc.")};
+const std::string SCILLA_IPC_SOCKET_PATH{
+    ReadConstantString("SCILLA_IPC_SOCKET_PATH", "node.jsonrpc.")};
 
 // Network composition constants
 const unsigned int COMM_SIZE{
@@ -424,6 +427,8 @@ const unsigned int RESUME_BLACKLIST_DELAY_IN_SECONDS{
     ReadConstantNumeric("RESUME_BLACKLIST_DELAY_IN_SECONDS", "node.recovery.")};
 const unsigned int INCRDB_DSNUMS_WITH_STATEDELTAS{
     ReadConstantNumeric("INCRDB_DSNUMS_WITH_STATEDELTAS", "node.recovery.")};
+const bool CONTRACT_STATES_MIGRATED{
+    ReadConstantString("CONTRACT_STATES_MIGRATED", "node.recovery.") == "true"};
 
 // Smart contract constants
 const bool ENABLE_SC{ReadConstantString("ENABLE_SC", "node.smart_contract.") ==
@@ -464,6 +469,10 @@ const string INPUT_CODE{
 const bool ENABLE_SCILLA_MULTI_VERSION{
     ReadConstantString("ENABLE_SCILLA_MULTI_VERSION", "node.smart_contract.") ==
     "true"};
+const string FIELDS_MAP_DEPTH_INDICATOR{
+    ReadConstantString("FIELDS_MAP_DEPTH_INDICATOR", "node.smart_contract.")};
+const bool LOG_SC{ReadConstantString("LOG_SC", "node.smart_contract.") ==
+                  "true"};
 
 // Test constants
 const bool ENABLE_CHECK_PERFORMANCE_LOG{

src/common/Constants.h

@@ -122,6 +122,8 @@ const std::string DS_BACKUP_MSG = "DS BACKUP NOW";
 
 const std::string dsNodeFile = "dsnodes.xml";
 
+const char SCILLA_INDEX_SEPARATOR = 0x16;
+
 const float ONE_HUNDRED_PERCENT = 100.f;
 
 // General constants
@@ -258,6 +260,7 @@ extern const unsigned int STATUS_RPC_PORT;
 extern const std::string IP_TO_BIND;  // Only for non-lookup nodes
 extern const bool ENABLE_STATUS_RPC;  //
 extern const unsigned int NUM_SHARD_PEER_TO_REVEAL;
+extern const std::string SCILLA_IPC_SOCKET_PATH;
 
 // Network composition constants
 extern const unsigned int COMM_SIZE;
@@ -323,6 +326,7 @@ extern const bool RECOVERY_TRIM_INCOMPLETED_BLOCK;
 extern const bool REJOIN_NODE_NOT_IN_NETWORK;
 extern const unsigned int RESUME_BLACKLIST_DELAY_IN_SECONDS;
 extern const unsigned int INCRDB_DSNUMS_WITH_STATEDELTAS;
+extern const bool CONTRACT_STATES_MIGRATED;
 
 // Smart contract constants
 extern const bool ENABLE_SC;
@@ -339,6 +343,8 @@ extern const std::string INPUT_MESSAGE_JSON;
 extern const std::string OUTPUT_JSON;
 extern const std::string INPUT_CODE;
 extern const bool ENABLE_SCILLA_MULTI_VERSION;
+extern const std::string FIELDS_MAP_DEPTH_INDICATOR;
+extern const bool LOG_SC;
 
 // Test constants
 extern const bool ENABLE_CHECK_PERFORMANCE_LOG;