Merge branch 'develop' into 05786-5396-misc-startup-improvements

Signed-off-by: Cody Littley <cody@swirldslabs.com>
hashgraph · Mar 30, 2023 · c244baa · c244baa
2 parents 84d4b64 + 6583dc1
commit c244baa
Show file tree

Hide file tree

Showing 46 changed files with 1,282 additions and 291 deletions.
diff --git a/hedera-node/configuration/dev/bootstrap.properties b/hedera-node/configuration/dev/bootstrap.properties
@@ -1 +1,2 @@
 bootstrap.throttleDefsJson.resource=throttles-dev.json
+virtualdatasource.jasperdbToMerkledb=true
diff --git a/hedera-node/configuration/dev/log4j2.xml b/hedera-node/configuration/dev/log4j2.xml
@@ -73,17 +73,19 @@
     <Logger name="com.swirlds" level="INFO" additivity="false">
       <AppenderRef ref="fileLog">
         <Filters>
-          <!-- JasperDB & Virtual Merkle -->
+          <!-- JasperDB / MerkleDb & Virtual Merkle -->
           <MarkerFilter marker="JASPER_DB"              onMatch="DENY" onMismatch="NEUTRAL"/>
+          <MarkerFilter marker="MERKLE_DB"              onMatch="DENY" onMismatch="NEUTRAL"/>
           <MarkerFilter marker="VIRTUAL_MERKLE_STATS"   onMatch="DENY" onMismatch="NEUTRAL"/>
           <MarkerFilter marker="STATE_HASH"             onMatch="DENY" onMismatch="NEUTRAL"/>
         </Filters>
       </AppenderRef>
 
       <AppenderRef ref="vMapLog">
         <Filters>
-          <!-- JasperDB & Virtual Merkle -->
+          <!-- JasperDB / MerkleDb & Virtual Merkle -->
           <MarkerFilter marker="JASPER_DB"              onMatch="ACCEPT" onMismatch="NEUTRAL"/>
+          <MarkerFilter marker="MERKLE_DB"              onMatch="ACCEPT" onMismatch="NEUTRAL"/>
           <MarkerFilter marker="VIRTUAL_MERKLE_STATS"   onMatch="ACCEPT" onMismatch="NEUTRAL"/>
           <MarkerFilter marker="DISABLED"               onMatch="DENY"   onMismatch="DENY" />
         </Filters>
@@ -183,8 +185,9 @@
         <MarkerFilter marker="MERKLE_GENERATION"      onMatch="DENY"   onMismatch="NEUTRAL"/>
         <MarkerFilter marker="MERKLE_LOCKS"           onMatch="DENY"   onMismatch="NEUTRAL"/>
 
-        <!-- JasperDB & Virtual Merkle -->
+        <!-- JasperDB / MerkleDd & Virtual Merkle -->
         <MarkerFilter marker="JASPER_DB"              onMatch="ACCEPT" onMismatch="NEUTRAL"/>
+        <MarkerFilter marker="MERKLE_DB"              onMatch="ACCEPT" onMismatch="NEUTRAL"/>
         <MarkerFilter marker="VIRTUAL_MERKLE_STATS"   onMatch="ACCEPT" onMismatch="NEUTRAL"/>
 
         <MarkerFilter marker="DISABLED"               onMatch="DENY"   onMismatch="DENY" />

diff --git a/hedera-node/configuration/dev/settings.txt b/hedera-node/configuration/dev/settings.txt
@@ -16,5 +16,6 @@ state.signedStateKeep,                         10
 useLoopbackIp,                                 false
 waitAtStartup,                                 false
 jasperDb.iteratorInputBufferBytes,             16777216
+merkleDb.iteratorInputBufferBytes,             16777216
 virtualMap.preferredFlushQueueSize,            10000
 transThrottle,                                 false
diff --git a/hedera-node/configuration/mainnet/log4j2.xml b/hedera-node/configuration/mainnet/log4j2.xml
@@ -72,17 +72,19 @@
     <Logger name="com.swirlds" level="INFO" additivity="false">
       <AppenderRef ref="fileLog">
         <Filters>
-          <!-- JasperDB & Virtual Merkle -->
+          <!-- JasperDB / MerkleDb & Virtual Merkle -->
           <MarkerFilter marker="JASPER_DB"              onMatch="DENY" onMismatch="NEUTRAL"/>
+          <MarkerFilter marker="MERKLE_DB"              onMatch="DENY" onMismatch="NEUTRAL"/>
           <MarkerFilter marker="VIRTUAL_MERKLE_STATS"   onMatch="DENY" onMismatch="NEUTRAL"/>
           <MarkerFilter marker="STATE_HASH"             onMatch="DENY" onMismatch="NEUTRAL"/>
         </Filters>
       </AppenderRef>
 
       <AppenderRef ref="vMapLog">
         <Filters>
-          <!-- JasperDB & Virtual Merkle -->
+          <!-- JasperDB / MerkleDb & Virtual Merkle -->
           <MarkerFilter marker="JASPER_DB"              onMatch="ACCEPT" onMismatch="NEUTRAL"/>
+          <MarkerFilter marker="MERKLE_DB"              onMatch="ACCEPT" onMismatch="NEUTRAL"/>
           <MarkerFilter marker="VIRTUAL_MERKLE_STATS"   onMatch="ACCEPT" onMismatch="NEUTRAL"/>
           <MarkerFilter marker="DISABLED"               onMatch="DENY"   onMismatch="DENY" />
         </Filters>
@@ -181,8 +183,9 @@
         <MarkerFilter marker="MERKLE_GENERATION"      onMatch="DENY"   onMismatch="NEUTRAL"/>
         <MarkerFilter marker="MERKLE_LOCKS"           onMatch="DENY"   onMismatch="NEUTRAL"/>
 
-        <!-- JasperDB & Virtual Merkle -->
+        <!-- JasperDB / MerkleDb & Virtual Merkle -->
         <MarkerFilter marker="JASPER_DB"              onMatch="ACCEPT" onMismatch="NEUTRAL"/>
+        <MarkerFilter marker="MERKLE_DB"              onMatch="ACCEPT" onMismatch="NEUTRAL"/>
         <MarkerFilter marker="VIRTUAL_MERKLE_STATS"   onMatch="ACCEPT" onMismatch="NEUTRAL"/>
 
         <MarkerFilter marker="DISABLED"               onMatch="DENY"   onMismatch="DENY" />

diff --git a/hedera-node/configuration/mainnet/settings.txt b/hedera-node/configuration/mainnet/settings.txt
@@ -23,8 +23,8 @@ state.signedStateKeep,                         10
 throttle7extra,                                0.5
 useLoopbackIp,                                 false
 waitAtStartup,                                 false
-jasperDb.storagePath,                          /opt/hgcapp/services-hedera/HapiApp2.0/data/saved
 jasperDb.iteratorInputBufferBytes,             16777216
+merkleDb.iteratorInputBufferBytes,             16777216
 virtualMap.preferredFlushQueueSize,            10000
 state.mainClassNameOverride,                   com.hedera.services.ServicesMain
 prometheusEndpointEnabled,                     true

diff --git a/hedera-node/configuration/preprod/log4j2.xml b/hedera-node/configuration/preprod/log4j2.xml
@@ -133,17 +133,19 @@
     <Logger name="com.swirlds" level="INFO" additivity="false">
       <AppenderRef ref="fileLog">
         <Filters>
-          <!-- JasperDB & Virtual Merkle -->
+          <!-- JasperDB / MerkleDb & Virtual Merkle -->
           <MarkerFilter marker="JASPER_DB"              onMatch="DENY" onMismatch="NEUTRAL"/>
+          <MarkerFilter marker="MERKLE_DB"              onMatch="DENY" onMismatch="NEUTRAL"/>
           <MarkerFilter marker="VIRTUAL_MERKLE_STATS"   onMatch="DENY" onMismatch="NEUTRAL"/>
           <MarkerFilter marker="STATE_HASH"             onMatch="DENY" onMismatch="NEUTRAL"/>
         </Filters>
       </AppenderRef>
 
       <AppenderRef ref="vMapLog">
         <Filters>
-          <!-- JasperDB & Virtual Merkle -->
+          <!-- JasperDB / MerkleDb & Virtual Merkle -->
           <MarkerFilter marker="JASPER_DB"              onMatch="ACCEPT" onMismatch="NEUTRAL"/>
+          <MarkerFilter marker="MERKLE_DB"              onMatch="ACCEPT" onMismatch="NEUTRAL"/>
           <MarkerFilter marker="VIRTUAL_MERKLE_STATS"   onMatch="ACCEPT" onMismatch="NEUTRAL"/>
           <MarkerFilter marker="DISABLED"               onMatch="DENY"   onMismatch="DENY" />
         </Filters>
@@ -243,8 +245,9 @@
         <MarkerFilter marker="MERKLE_GENERATION"      onMatch="DENY"   onMismatch="NEUTRAL"/>
         <MarkerFilter marker="MERKLE_LOCKS"           onMatch="DENY"   onMismatch="NEUTRAL"/>
 
-        <!-- JasperDB & Virtual Merkle -->
+        <!-- JasperDB / MerkleDb & Virtual Merkle -->
         <MarkerFilter marker="JASPER_DB"              onMatch="ACCEPT" onMismatch="NEUTRAL"/>
+        <MarkerFilter marker="MERKLE_DB"              onMatch="ACCEPT" onMismatch="NEUTRAL"/>
         <MarkerFilter marker="VIRTUAL_MERKLE_STATS"   onMatch="ACCEPT" onMismatch="NEUTRAL"/>
 
         <MarkerFilter marker="DISABLED"               onMatch="DENY"   onMismatch="DENY" />

diff --git a/hedera-node/configuration/preprod/settings.txt b/hedera-node/configuration/preprod/settings.txt
@@ -23,8 +23,8 @@ state.signedStateKeep,                         10
 throttle7extra,                                0.5
 useLoopbackIp,                                 false
 waitAtStartup,                                 false
-jasperDb.storagePath,                          /opt/hgcapp/services-hedera/HapiApp2.0/data/saved
 jasperDb.iteratorInputBufferBytes,             16777216
+merkleDb.iteratorInputBufferBytes,             16777216
 virtualMap.preferredFlushQueueSize,            10000
 prometheusEndpointEnabled,                     true
 state.mainClassNameOverride,                   com.hedera.services.ServicesMain

diff --git a/hedera-node/configuration/previewnet/log4j2.xml b/hedera-node/configuration/previewnet/log4j2.xml
@@ -73,17 +73,19 @@
     <Logger name="com.swirlds" level="INFO" additivity="false">
       <AppenderRef ref="fileLog">
         <Filters>
-          <!-- JasperDB & Virtual Merkle -->
+          <!-- JasperDB / MerkleDb & Virtual Merkle -->
           <MarkerFilter marker="JASPER_DB"              onMatch="DENY" onMismatch="NEUTRAL"/>
+          <MarkerFilter marker="MERKLE_DB"              onMatch="DENY" onMismatch="NEUTRAL"/>
           <MarkerFilter marker="VIRTUAL_MERKLE_STATS"   onMatch="DENY" onMismatch="NEUTRAL"/>
           <MarkerFilter marker="STATE_HASH"             onMatch="DENY" onMismatch="NEUTRAL"/>
         </Filters>
       </AppenderRef>
 
       <AppenderRef ref="vMapLog">
         <Filters>
-          <!-- JasperDB & Virtual Merkle -->
+          <!-- JasperDB / MerkleDb & Virtual Merkle -->
           <MarkerFilter marker="JASPER_DB"              onMatch="ACCEPT" onMismatch="NEUTRAL"/>
+          <MarkerFilter marker="MERKLE_DB"              onMatch="ACCEPT" onMismatch="NEUTRAL"/>
           <MarkerFilter marker="VIRTUAL_MERKLE_STATS"   onMatch="ACCEPT" onMismatch="NEUTRAL"/>
           <MarkerFilter marker="DISABLED"               onMatch="DENY"   onMismatch="DENY" />
         </Filters>
@@ -183,8 +185,9 @@
         <MarkerFilter marker="MERKLE_GENERATION"      onMatch="DENY"   onMismatch="NEUTRAL"/>
         <MarkerFilter marker="MERKLE_LOCKS"           onMatch="DENY"   onMismatch="NEUTRAL"/>
 
-        <!-- JasperDB & Virtual Merkle -->
+        <!-- JasperDB / MerkleDb & Virtual Merkle -->
         <MarkerFilter marker="JASPER_DB"              onMatch="ACCEPT" onMismatch="NEUTRAL"/>
+        <MarkerFilter marker="MERKLE_DB"              onMatch="ACCEPT" onMismatch="NEUTRAL"/>
         <MarkerFilter marker="VIRTUAL_MERKLE_STATS"   onMatch="ACCEPT" onMismatch="NEUTRAL"/>
 
         <MarkerFilter marker="DISABLED"               onMatch="DENY"   onMismatch="DENY" />

diff --git a/hedera-node/configuration/previewnet/settings.txt b/hedera-node/configuration/previewnet/settings.txt
@@ -23,8 +23,8 @@ state.signedStateKeep,                         10
 throttle7extra,                                0.5
 useLoopbackIp,                                 false
 waitAtStartup,                                 false
-jasperDb.storagePath,                          /opt/hgcapp/services-hedera/HapiApp2.0/data/saved
 jasperDb.iteratorInputBufferBytes,             16777216
+merkleDb.iteratorInputBufferBytes,             16777216
 virtualMap.preferredFlushQueueSize,            10000
 state.mainClassNameOverride,                   com.hedera.services.ServicesMain
 prometheusEndpointEnabled,                     true

diff --git a/hedera-node/configuration/testnet/log4j2.xml b/hedera-node/configuration/testnet/log4j2.xml
@@ -73,17 +73,19 @@
     <Logger name="com.swirlds" level="INFO" additivity="false">
       <AppenderRef ref="fileLog">
         <Filters>
-          <!-- JasperDB & Virtual Merkle -->
+          <!-- JasperDB / MerkleDb & Virtual Merkle -->
           <MarkerFilter marker="JASPER_DB"              onMatch="DENY" onMismatch="NEUTRAL"/>
+          <MarkerFilter marker="MERKLE_DB"              onMatch="DENY" onMismatch="NEUTRAL"/>
           <MarkerFilter marker="VIRTUAL_MERKLE_STATS"   onMatch="DENY" onMismatch="NEUTRAL"/>
           <MarkerFilter marker="STATE_HASH"             onMatch="DENY" onMismatch="NEUTRAL"/>
         </Filters>
       </AppenderRef>
 
       <AppenderRef ref="vMapLog">
         <Filters>
-          <!-- JasperDB & Virtual Merkle -->
+          <!-- JasperDB / MerkleDb & Virtual Merkle -->
           <MarkerFilter marker="JASPER_DB"              onMatch="ACCEPT" onMismatch="NEUTRAL"/>
+          <MarkerFilter marker="MERKLE_DB"              onMatch="ACCEPT" onMismatch="NEUTRAL"/>
           <MarkerFilter marker="VIRTUAL_MERKLE_STATS"   onMatch="ACCEPT" onMismatch="NEUTRAL"/>
           <MarkerFilter marker="DISABLED"               onMatch="DENY"   onMismatch="DENY" />
         </Filters>
@@ -183,8 +185,9 @@
         <MarkerFilter marker="MERKLE_GENERATION"      onMatch="DENY"   onMismatch="NEUTRAL"/>
         <MarkerFilter marker="MERKLE_LOCKS"           onMatch="DENY"   onMismatch="NEUTRAL"/>
 
-        <!-- JasperDB & Virtual Merkle -->
+        <!-- JasperDB / MerkleDb & Virtual Merkle -->
         <MarkerFilter marker="JASPER_DB"              onMatch="ACCEPT" onMismatch="NEUTRAL"/>
+        <MarkerFilter marker="MERKLE_DB"              onMatch="ACCEPT" onMismatch="NEUTRAL"/>
         <MarkerFilter marker="VIRTUAL_MERKLE_STATS"   onMatch="ACCEPT" onMismatch="NEUTRAL"/>
 
         <MarkerFilter marker="DISABLED"               onMatch="DENY"   onMismatch="DENY" />

diff --git a/hedera-node/configuration/testnet/settings.txt b/hedera-node/configuration/testnet/settings.txt
@@ -24,8 +24,8 @@ state.signedStateKeep,                         10
 throttle7extra,                                0.5
 useLoopbackIp,                                 false
 waitAtStartup,                                 false
-jasperDb.storagePath,                          /opt/hgcapp/services-hedera/HapiApp2.0/data/saved
 jasperDb.iteratorInputBufferBytes,             16777216
+merkleDb.iteratorInputBufferBytes,             16777216
 virtualMap.preferredFlushQueueSize,            10000
 state.mainClassNameOverride,                   com.hedera.services.ServicesMain
 chatter.useChatter,                            false

diff --git a/hedera-node/hedera-app/src/test/resources/bootstrap.properties b/hedera-node/hedera-app/src/test/resources/bootstrap.properties
@@ -211,4 +211,4 @@ hedera.prefetch.threadPoolSize=4
 hedera.prefetch.codeCacheTtlSecs=600
 utilPrng.isEnabled=true
 tokens.autoCreations.isEnabled=true
-virtualdatasource.jasperdbToMerkledb=false
+virtualdatasource.jasperdbToMerkledb=true
diff --git a/...vice/src/jmh/java/com/hedera/node/app/service/mono/contracts/SizeLimitedStorageBench.java b/...vice/src/jmh/java/com/hedera/node/app/service/mono/contracts/SizeLimitedStorageBench.java
@@ -130,9 +130,9 @@ public void simulateContractTransaction() {
     private void registerConstructables() {
         try {
             Constructables.registerForAccounts();
-            Constructables.registerForJasperDb();
             Constructables.registerForMerkleMap();
             Constructables.registerForVirtualMap();
+            Constructables.registerForVirtualDataSource();
             Constructables.registerForContractStorage();
         } catch (final ConstructableRegistryException e) {
             throw new IllegalStateException(e);

diff --git a/...dera-mono-service/src/jmh/java/com/hedera/node/app/service/mono/setup/Constructables.java b/...dera-mono-service/src/jmh/java/com/hedera/node/app/service/mono/setup/Constructables.java
@@ -37,6 +37,7 @@
 import com.swirlds.merkle.map.MerkleMap;
 import com.swirlds.merkle.tree.MerkleBinaryTree;
 import com.swirlds.merkle.tree.MerkleTreeInternalNode;
+import com.swirlds.merkledb.MerkleDbDataSourceBuilder;
 import com.swirlds.virtualmap.VirtualMap;
 import com.swirlds.virtualmap.internal.cache.VirtualNodeCache;
 import com.swirlds.virtualmap.internal.merkle.VirtualMapState;
@@ -129,8 +130,11 @@ public static void registerForVirtualMap() throws ConstructableRegistryException
                 .registerConstructable(new ClassConstructorPair(VirtualNodeCache.class, VirtualNodeCache::new));
     }
 
-    public static void registerForJasperDb() throws ConstructableRegistryException {
+    public static void registerForVirtualDataSource() throws ConstructableRegistryException {
         ConstructableRegistry.getInstance()
                 .registerConstructable(new ClassConstructorPair(JasperDbBuilder.class, JasperDbBuilder::new));
+        ConstructableRegistry.getInstance()
+                .registerConstructable(
+                        new ClassConstructorPair(MerkleDbDataSourceBuilder.class, MerkleDbDataSourceBuilder::new));
     }
 }
diff --git a/...est/java/com/hedera/node/app/service/mono/context/properties/BootstrapPropertiesTest.java b/...est/java/com/hedera/node/app/service/mono/context/properties/BootstrapPropertiesTest.java
@@ -504,7 +504,7 @@ class BootstrapPropertiesTest {
             entry(HEDERA_RECORD_STREAM_COMPRESS_FILES_ON_CREATION, true),
             entry(TOKENS_AUTO_CREATIONS_ENABLED, true),
             entry(WORKFLOWS_ENABLED, Set.of()),
-            entry(VIRTUALDATASOURCE_JASPERDB_TO_MERKLEDB, false));
+            entry(VIRTUALDATASOURCE_JASPERDB_TO_MERKLEDB, true));
 
     @Test
     void containsProperty() {

diff --git a/hedera-node/hedera-mono-service/src/test/resources/bootstrap.properties b/hedera-node/hedera-mono-service/src/test/resources/bootstrap.properties
@@ -211,4 +211,4 @@ hedera.prefetch.threadPoolSize=4
 hedera.prefetch.codeCacheTtlSecs=600
 utilPrng.isEnabled=true
 tokens.autoCreations.isEnabled=true
-virtualdatasource.jasperdbToMerkledb=false
+virtualdatasource.jasperdbToMerkledb=true
diff --git a/hedera-node/hedera-mono-service/src/test/resources/bootstrap/standard.properties b/hedera-node/hedera-mono-service/src/test/resources/bootstrap/standard.properties
@@ -209,4 +209,4 @@ stats.runningAvgHalfLifeSecs=10.0
 stats.speedometerHalfLifeSecs=10.0
 utilPrng.isEnabled=true
 tokens.autoCreations.isEnabled=true
-virtualdatasource.jasperdbToMerkledb=false
+virtualdatasource.jasperdbToMerkledb=true