diff --git a/hedera-node/configuration/dev/bootstrap.properties b/hedera-node/configuration/dev/bootstrap.properties index 34525d2b7a8e..eeb1787fc55b 100644 --- a/hedera-node/configuration/dev/bootstrap.properties +++ b/hedera-node/configuration/dev/bootstrap.properties @@ -1 +1,2 @@ bootstrap.throttleDefsJson.resource=throttles-dev.json +virtualdatasource.jasperdbToMerkledb=true diff --git a/hedera-node/configuration/dev/log4j2.xml b/hedera-node/configuration/dev/log4j2.xml index acbfe5d1d54c..d91738da6be3 100644 --- a/hedera-node/configuration/dev/log4j2.xml +++ b/hedera-node/configuration/dev/log4j2.xml @@ -73,8 +73,9 @@ - + + @@ -82,8 +83,9 @@ - + + @@ -183,8 +185,9 @@ - + + diff --git a/hedera-node/configuration/dev/settings.txt b/hedera-node/configuration/dev/settings.txt index 1d6811b0e1a5..108a14cb28fd 100644 --- a/hedera-node/configuration/dev/settings.txt +++ b/hedera-node/configuration/dev/settings.txt @@ -16,5 +16,6 @@ state.signedStateKeep, 10 useLoopbackIp, false waitAtStartup, false jasperDb.iteratorInputBufferBytes, 16777216 +merkleDb.iteratorInputBufferBytes, 16777216 virtualMap.preferredFlushQueueSize, 10000 transThrottle, false \ No newline at end of file diff --git a/hedera-node/configuration/mainnet/log4j2.xml b/hedera-node/configuration/mainnet/log4j2.xml index 47011e950a43..0a7f268154e7 100644 --- a/hedera-node/configuration/mainnet/log4j2.xml +++ b/hedera-node/configuration/mainnet/log4j2.xml @@ -72,8 +72,9 @@ - + + @@ -81,8 +82,9 @@ - + + @@ -181,8 +183,9 @@ - + + diff --git a/hedera-node/configuration/mainnet/settings.txt b/hedera-node/configuration/mainnet/settings.txt index 32eb1e96669c..94a518e7073c 100644 --- a/hedera-node/configuration/mainnet/settings.txt +++ b/hedera-node/configuration/mainnet/settings.txt @@ -23,8 +23,8 @@ state.signedStateKeep, 10 throttle7extra, 0.5 useLoopbackIp, false waitAtStartup, false -jasperDb.storagePath, /opt/hgcapp/services-hedera/HapiApp2.0/data/saved jasperDb.iteratorInputBufferBytes, 16777216 +merkleDb.iteratorInputBufferBytes, 16777216 virtualMap.preferredFlushQueueSize, 10000 state.mainClassNameOverride, com.hedera.services.ServicesMain prometheusEndpointEnabled, true diff --git a/hedera-node/configuration/preprod/log4j2.xml b/hedera-node/configuration/preprod/log4j2.xml index 8f75276a0043..48fd7e55cdb3 100644 --- a/hedera-node/configuration/preprod/log4j2.xml +++ b/hedera-node/configuration/preprod/log4j2.xml @@ -133,8 +133,9 @@ - + + @@ -142,8 +143,9 @@ - + + @@ -243,8 +245,9 @@ - + + diff --git a/hedera-node/configuration/preprod/settings.txt b/hedera-node/configuration/preprod/settings.txt index 2dfcc2f97d13..49d3a05d973f 100644 --- a/hedera-node/configuration/preprod/settings.txt +++ b/hedera-node/configuration/preprod/settings.txt @@ -23,8 +23,8 @@ state.signedStateKeep, 10 throttle7extra, 0.5 useLoopbackIp, false waitAtStartup, false -jasperDb.storagePath, /opt/hgcapp/services-hedera/HapiApp2.0/data/saved jasperDb.iteratorInputBufferBytes, 16777216 +merkleDb.iteratorInputBufferBytes, 16777216 virtualMap.preferredFlushQueueSize, 10000 prometheusEndpointEnabled, true state.mainClassNameOverride, com.hedera.services.ServicesMain diff --git a/hedera-node/configuration/previewnet/log4j2.xml b/hedera-node/configuration/previewnet/log4j2.xml index 72d2ce969f2e..81ce6230aaad 100644 --- a/hedera-node/configuration/previewnet/log4j2.xml +++ b/hedera-node/configuration/previewnet/log4j2.xml @@ -73,8 +73,9 @@ - + + @@ -82,8 +83,9 @@ - + + @@ -183,8 +185,9 @@ - + + diff --git a/hedera-node/configuration/previewnet/settings.txt b/hedera-node/configuration/previewnet/settings.txt index 689997563c47..1cc4f71484fd 100644 --- a/hedera-node/configuration/previewnet/settings.txt +++ b/hedera-node/configuration/previewnet/settings.txt @@ -23,8 +23,8 @@ state.signedStateKeep, 10 throttle7extra, 0.5 useLoopbackIp, false waitAtStartup, false -jasperDb.storagePath, /opt/hgcapp/services-hedera/HapiApp2.0/data/saved jasperDb.iteratorInputBufferBytes, 16777216 +merkleDb.iteratorInputBufferBytes, 16777216 virtualMap.preferredFlushQueueSize, 10000 state.mainClassNameOverride, com.hedera.services.ServicesMain prometheusEndpointEnabled, true diff --git a/hedera-node/configuration/testnet/log4j2.xml b/hedera-node/configuration/testnet/log4j2.xml index 72d2ce969f2e..81ce6230aaad 100644 --- a/hedera-node/configuration/testnet/log4j2.xml +++ b/hedera-node/configuration/testnet/log4j2.xml @@ -73,8 +73,9 @@ - + + @@ -82,8 +83,9 @@ - + + @@ -183,8 +185,9 @@ - + + diff --git a/hedera-node/configuration/testnet/settings.txt b/hedera-node/configuration/testnet/settings.txt index 9a7d2352c369..053a0220483c 100644 --- a/hedera-node/configuration/testnet/settings.txt +++ b/hedera-node/configuration/testnet/settings.txt @@ -24,8 +24,8 @@ state.signedStateKeep, 10 throttle7extra, 0.5 useLoopbackIp, false waitAtStartup, false -jasperDb.storagePath, /opt/hgcapp/services-hedera/HapiApp2.0/data/saved jasperDb.iteratorInputBufferBytes, 16777216 +merkleDb.iteratorInputBufferBytes, 16777216 virtualMap.preferredFlushQueueSize, 10000 state.mainClassNameOverride, com.hedera.services.ServicesMain chatter.useChatter, false diff --git a/hedera-node/hedera-app/src/test/resources/bootstrap.properties b/hedera-node/hedera-app/src/test/resources/bootstrap.properties index 01348f2507a8..3ff4cdb38f10 100644 --- a/hedera-node/hedera-app/src/test/resources/bootstrap.properties +++ b/hedera-node/hedera-app/src/test/resources/bootstrap.properties @@ -211,4 +211,4 @@ hedera.prefetch.threadPoolSize=4 hedera.prefetch.codeCacheTtlSecs=600 utilPrng.isEnabled=true tokens.autoCreations.isEnabled=true -virtualdatasource.jasperdbToMerkledb=false +virtualdatasource.jasperdbToMerkledb=true diff --git a/hedera-node/hedera-mono-service/src/jmh/java/com/hedera/node/app/service/mono/contracts/SizeLimitedStorageBench.java b/hedera-node/hedera-mono-service/src/jmh/java/com/hedera/node/app/service/mono/contracts/SizeLimitedStorageBench.java index 4df222d9e1cf..6ec091da2b5e 100644 --- a/hedera-node/hedera-mono-service/src/jmh/java/com/hedera/node/app/service/mono/contracts/SizeLimitedStorageBench.java +++ b/hedera-node/hedera-mono-service/src/jmh/java/com/hedera/node/app/service/mono/contracts/SizeLimitedStorageBench.java @@ -130,9 +130,9 @@ public void simulateContractTransaction() { private void registerConstructables() { try { Constructables.registerForAccounts(); - Constructables.registerForJasperDb(); Constructables.registerForMerkleMap(); Constructables.registerForVirtualMap(); + Constructables.registerForVirtualDataSource(); Constructables.registerForContractStorage(); } catch (final ConstructableRegistryException e) { throw new IllegalStateException(e); diff --git a/hedera-node/hedera-mono-service/src/jmh/java/com/hedera/node/app/service/mono/setup/Constructables.java b/hedera-node/hedera-mono-service/src/jmh/java/com/hedera/node/app/service/mono/setup/Constructables.java index ff3a7741822c..fd289c219c3d 100644 --- a/hedera-node/hedera-mono-service/src/jmh/java/com/hedera/node/app/service/mono/setup/Constructables.java +++ b/hedera-node/hedera-mono-service/src/jmh/java/com/hedera/node/app/service/mono/setup/Constructables.java @@ -37,6 +37,7 @@ import com.swirlds.merkle.map.MerkleMap; import com.swirlds.merkle.tree.MerkleBinaryTree; import com.swirlds.merkle.tree.MerkleTreeInternalNode; +import com.swirlds.merkledb.MerkleDbDataSourceBuilder; import com.swirlds.virtualmap.VirtualMap; import com.swirlds.virtualmap.internal.cache.VirtualNodeCache; import com.swirlds.virtualmap.internal.merkle.VirtualMapState; @@ -129,8 +130,11 @@ public static void registerForVirtualMap() throws ConstructableRegistryException .registerConstructable(new ClassConstructorPair(VirtualNodeCache.class, VirtualNodeCache::new)); } - public static void registerForJasperDb() throws ConstructableRegistryException { + public static void registerForVirtualDataSource() throws ConstructableRegistryException { ConstructableRegistry.getInstance() .registerConstructable(new ClassConstructorPair(JasperDbBuilder.class, JasperDbBuilder::new)); + ConstructableRegistry.getInstance() + .registerConstructable( + new ClassConstructorPair(MerkleDbDataSourceBuilder.class, MerkleDbDataSourceBuilder::new)); } } diff --git a/hedera-node/hedera-mono-service/src/test/java/com/hedera/node/app/service/mono/context/properties/BootstrapPropertiesTest.java b/hedera-node/hedera-mono-service/src/test/java/com/hedera/node/app/service/mono/context/properties/BootstrapPropertiesTest.java index 4fea65652e34..eea425a7d45a 100644 --- a/hedera-node/hedera-mono-service/src/test/java/com/hedera/node/app/service/mono/context/properties/BootstrapPropertiesTest.java +++ b/hedera-node/hedera-mono-service/src/test/java/com/hedera/node/app/service/mono/context/properties/BootstrapPropertiesTest.java @@ -504,7 +504,7 @@ class BootstrapPropertiesTest { entry(HEDERA_RECORD_STREAM_COMPRESS_FILES_ON_CREATION, true), entry(TOKENS_AUTO_CREATIONS_ENABLED, true), entry(WORKFLOWS_ENABLED, Set.of()), - entry(VIRTUALDATASOURCE_JASPERDB_TO_MERKLEDB, false)); + entry(VIRTUALDATASOURCE_JASPERDB_TO_MERKLEDB, true)); @Test void containsProperty() { diff --git a/hedera-node/hedera-mono-service/src/test/resources/bootstrap.properties b/hedera-node/hedera-mono-service/src/test/resources/bootstrap.properties index 8573b333106d..64edbe7114d0 100644 --- a/hedera-node/hedera-mono-service/src/test/resources/bootstrap.properties +++ b/hedera-node/hedera-mono-service/src/test/resources/bootstrap.properties @@ -211,4 +211,4 @@ hedera.prefetch.threadPoolSize=4 hedera.prefetch.codeCacheTtlSecs=600 utilPrng.isEnabled=true tokens.autoCreations.isEnabled=true -virtualdatasource.jasperdbToMerkledb=false +virtualdatasource.jasperdbToMerkledb=true diff --git a/hedera-node/hedera-mono-service/src/test/resources/bootstrap/standard.properties b/hedera-node/hedera-mono-service/src/test/resources/bootstrap/standard.properties index 7f0308bfb7ad..11a8cb181e05 100644 --- a/hedera-node/hedera-mono-service/src/test/resources/bootstrap/standard.properties +++ b/hedera-node/hedera-mono-service/src/test/resources/bootstrap/standard.properties @@ -209,4 +209,4 @@ stats.runningAvgHalfLifeSecs=10.0 stats.speedometerHalfLifeSecs=10.0 utilPrng.isEnabled=true tokens.autoCreations.isEnabled=true -virtualdatasource.jasperdbToMerkledb=false +virtualdatasource.jasperdbToMerkledb=true diff --git a/hedera-node/test-clients/src/main/resource/bootstrap.properties b/hedera-node/test-clients/src/main/resource/bootstrap.properties index c1b1f1d87b70..acca391fd54e 100644 --- a/hedera-node/test-clients/src/main/resource/bootstrap.properties +++ b/hedera-node/test-clients/src/main/resource/bootstrap.properties @@ -16,13 +16,17 @@ accounts.addressBookAdmin=55 accounts.exchangeRatesAdmin=57 accounts.feeSchedulesAdmin=56 accounts.freezeAdmin=58 +accounts.lastThrottleExempt=100 accounts.nodeRewardAccount=801 accounts.stakingRewardAccount=800 +accounts.storeOnDisk=false accounts.systemAdmin=50 accounts.systemDeleteAdmin=59 accounts.systemUndeleteAdmin=60 accounts.treasury=2 +autorenew.grantFreeRenewals=false entities.maxLifetime=3153600000 +entities.systemDeletable=FILE files.addressBook=101 files.networkProperties=121 files.exchangeRates=112 @@ -31,6 +35,7 @@ files.hapiPermissions=122 files.nodeDetails=102 files.softwareUpdateRange=150-159 files.throttleDefinitions=123 +hedera.firstUserEntity=1001 hedera.realm=0 hedera.shard=0 ledger.id=0x03 @@ -38,47 +43,64 @@ ledger.numSystemAccounts=100 ledger.totalTinyBarFloat=5000000000000000000 staking.periodMins=1 staking.rewardHistory.numStoredPeriods=3 +staking.startupHelper.recompute=NODE_STAKES,PENDING_REWARDS +tokens.storeRelsOnDisk=false hedera.workflows.enabled= # Dynamic properties accounts.maxNumber=5_000_000 autoCreation.enabled=true lazyCreation.enabled=true -cryptoCreateWithAlias.enabled=true +cryptoCreateWithAlias.enabled=false entities.limitTokenAssociations=false balances.exportDir.path=/opt/hgcapp/accountBalances/ balances.exportEnabled=true balances.exportPeriodSecs=900 balances.exportTokenBalances=true balances.nodeBalanceWarningThreshold=0 +balances.compressOnCreation=false cache.records.ttl=180 contracts.allowAutoAssociations=true contracts.allowCreate2=true +contracts.chainId=295 contracts.defaultLifetime=7890000 +contracts.enforceCreationThrottle=false contracts.evm.version=v0.34 contracts.evm.version.dynamic=false +contracts.permittedDelegateCallers=1062787,1461860 contracts.freeStorageTierLimit=100 contracts.itemizeStorageFees=true +contracts.keys.legacyActivations=1058134by[1062784] +contracts.knownBlockHash= contracts.localCall.estRetBytes=32 contracts.maxGasPerSec=15000000 contracts.maxKvPairs.aggregate=500_000_000 contracts.maxKvPairs.individual=163_840 contracts.maxNumber=5_000_000 -contracts.chainId=295 -contracts.throttle.throttleByGas=false contracts.maxRefundPercentOfGasLimit=20 -contracts.scheduleThrottleMaxGasLimit=5000000 -contracts.sidecars= -contract.storageSlotPriceTiers=0til100M,2000til450M -contracts.redirectTokenCalls=true -contracts.precompile.htsDefaultGasCost=10000 +contracts.precompile.exchangeRateGasCost=100 contracts.precompile.exportRecordResults=true +contracts.precompile.htsDefaultGasCost=10000 contracts.precompile.htsEnableTokenCreate=true +# Current implementation requires two storage get()'s to remove a slot +expiry.minCycleEntryCapacity=ACCOUNTS_GET,ACCOUNTS_GET_FOR_MODIFY,STORAGE_GET,STORAGE_GET,STORAGE_REMOVE,STORAGE_PUT +expiry.throttleResource=expiry-throttle.json +contracts.redirectTokenCalls=true +contracts.referenceSlotLifetime=31536000 +contracts.scheduleThrottleMaxGasLimit=5000000 +contracts.sidecars=CONTRACT_STATE_CHANGE,CONTRACT_BYTECODE,CONTRACT_ACTION +contracts.sidecarValidationEnabled=false +contract.storageSlotPriceTiers=0til100M,2000til450M +contracts.throttle.throttleByGas=false contracts.precompile.atomicCryptoTransfer.enabled=true fees.minCongestionPeriod=60 fees.percentCongestionMultipliers=90,10x,95,25x,99,100x +# Disable entity utilization congestion pricing by default +fees.percentUtilizationScaleFactors=DEFAULT(0,1:1) fees.tokenTransferUsageMultiplier=380 files.maxNumber=1_000_000 files.maxSizeKb=1024 +hedera.recordStream.enableTraceabilityMigration=true +hedera.recordStream.sidecarMaxSizeMb=256 hedera.transaction.maxMemoUtf8Bytes=100 hedera.transaction.maxValidDuration=180 hedera.transaction.minValidDuration=15 @@ -98,7 +120,6 @@ ledger.autoRenewPeriod.minDuration=6999999 ledger.changeHistorian.memorySecs=20 ledger.xferBalanceChanges.maxLen=20 ledger.fundingAccount=98 -ledger.maxAccountNum=100000000 ledger.transfers.maxLen=10 ledger.tokenTransfers.maxLen=10 ledger.nftTransfers.maxLen=10 @@ -111,11 +132,22 @@ scheduling.longTermEnabled=false scheduling.maxNumber=10_000_000 scheduling.maxTxnPerSecond=100 scheduling.maxExpirationFutureSeconds=5356800 -sigs.expandFromLastSignedState=true +sigs.expandFromImmutableState=true +staking.fees.nodeRewardPercentage=10 +staking.fees.stakingRewardPercentage=10 +staking.maxDailyStakeRewardThPerH=100 +traceability.maxExportsPerConsSec=10 +traceability.minFreeToUsedGasThrottleRatio=9 +# A possibly empty comma-separated : list; default ratio is 4 +staking.nodeMaxToMinStakeRatios= +staking.isEnabled=true +staking.rewardRate=100_000_000_000 +staking.requireMinStakeToReward=false +staking.startThreshold=100_000_000 tokens.maxAggregateRels=10_000_000 tokens.maxNumber=1_000_000 -tokens.maxRelsPerInfoQuery=1000 tokens.maxPerAccount=1000 +tokens.maxRelsPerInfoQuery=1000 tokens.maxSymbolUtf8Bytes=100 tokens.maxTokenNameUtf8Bytes=100 tokens.maxCustomFeesAllowed=10 @@ -129,6 +161,7 @@ tokens.nfts.maxAllowedMints=5000000 tokens.nfts.maxQueryRange=100 tokens.nfts.mintThrottleScaleFactor=5:2 tokens.nfts.useTreasuryWildcards=true +tokens.nfts.useVirtualMerkle=false topics.maxNumber=1_000_000 upgrade.artifacts.path=/opt/hgcapp/services-hedera/HapiApp2.0/data/upgrade/current # Node properties (can be overridden via data/config/node.properties) @@ -143,9 +176,13 @@ hedera.exportAccountsOnStartup=false hedera.profiles.active=PROD hedera.recordStream.isEnabled=true hedera.recordStream.logDir=/opt/hgcapp/recordStreams -hedera.recordStream.sidecarDir= +# hedera.recordStream.sidecarDir is specified relative to hedera.recordStream.logDir; blank==same dir +hedera.recordStream.sidecarDir=sidecar hedera.recordStream.logPeriod=2 hedera.recordStream.queueCapacity=5000 +hedera.recordStream.recordFileVersion=6 +hedera.recordStream.signatureFileVersion=6 +hedera.recordStream.logEveryTransaction=false hedera.recordStream.compressFilesOnCreation=false iss.resetPeriod=60 iss.roundsToLog=5000 @@ -162,6 +199,8 @@ netty.startRetryIntervalMs=1000 netty.tlsCrt.path=hedera.crt netty.tlsKey.path=hedera.key queries.blob.lookupRetries=3 +stats.consThrottlesToSample=,ThroughputLimits,CreationLimits +stats.hapiThrottlesToSample=,ThroughputLimits,OffHeapQueryLimits,CreationLimits,FreeQueryLimits stats.executionTimesToTrack=0 stats.entityUtils.gaugeUpdateIntervalMs=3000 stats.hapiOps.speedometerUpdateIntervalMs=3000 @@ -171,12 +210,6 @@ stats.speedometerHalfLifeSecs=10.0 hedera.prefetch.queueCapacity=70000 hedera.prefetch.threadPoolSize=4 hedera.prefetch.codeCacheTtlSecs=600 -staking.fees.nodeRewardPercentage=10 -staking.fees.stakingRewardPercentage=10 -staking.isEnabled=true -staking.maxDailyStakeRewardThPerH=100 -staking.rewardRate=100_000_000_000 -staking.startThreshold=100_000_000 utilPrng.isEnabled=true tokens.autoCreations.isEnabled=true -virtualdatasource.jasperdbToMerkledb=false \ No newline at end of file +virtualdatasource.jasperdbToMerkledb=false diff --git a/platform-sdk/docs/threads/eventflow-threads.drawio.svg b/platform-sdk/docs/threads/eventflow-threads.drawio.svg index 24e735d2bc8c..743392c6c51c 100644 --- a/platform-sdk/docs/threads/eventflow-threads.drawio.svg +++ b/platform-sdk/docs/threads/eventflow-threads.drawio.svg @@ -1,4 +1,4 @@ -
SwirldsPlatform
SwirldsPlatform
createPrioritySystemTransaction
createPrioritySystemTransaction
SignatureTransmitter
SignatureTransmitter
Template
Template
ClassName
ClassName
ClassName
ClassName
PreConsensusEventHandler
PreConsensusEventHandler
PreConsensusEventObserver
PreConsensusEventObserver
EventObserverDispatcher
EventObserverDispatcher
EventIntake
EventIntake
Consumer<GossipEvent>
Consumer<GossipEvent>
EventValidator
EventValidator
EventTaskDispatcher
EventTaskDispatcher
Queue Thread Name
(thread-name)
Queue Thread Name...
Note: the name in the parenthesis should match the value set with QueueThreadConfiguration::setThreadName
Note: the name in the parenthes...
EventIntakeTask
EventIntakeTask
Event Intake Queue
(event-intake)
Event Intake Queue...
EventCreator
EventCreator
createEvent
createEvent
pathway not used with OOO gossip
pathway not used with OOO gossip
GossipEvent
GossipEvent
EventImpl
EventImpl
addUnlinkedEvent
addUnlinkedEvent
ConsensusRounds
ConsensusRounds
addEvent
addEvent
dispatchTask
dispatchTask
Conditional
Conditional
instanceof CreateEventTask
instanceof CreateEvent...
instanceof ValidEvent
instanceof ValidEvent
instanceof GossipEvent
instanceof GossipEvent
CreateEventTask
CreateEventTask
GossipEvent
GossipEvent
GossipEvent
GossipEvent
validateEvent
validateEvent
PreConsensusEventObserver
PreConsensusEventObserver
preConsensusEvent
preConsensusEvent
EventImpl
EventImpl
EventImpl
EventImpl
preConsensusEvent
preConsensusEvent
PreConsensus Event Queue
(thread-curr)
PreConsensus Event Queue...
SwirldStateManagerImpl
SwirldStateManagerImpl
SwirldStateManager
SwirldStateManager
handlePreConsensusEvent
handlePreConsensusEvent
EventImpl
EventImpl
preHandle
preHandle
EventImpl
EventImpl
DefaultStateManagementComponent
DefaultStateManagementComponent
ConsensusHashManager
ConsensusHashManager
SignedStateManager
SignedStateManager
postConsensusSignatureObserver
postConsensusSignatureObserver
signature info
signature info
preConsensusSignatureObserver
preConsensusSignatureObserver
signature info
signature info
TransactionHandler
TransactionHandler
preHandle
preHandle
EventImpl, state
EventImpl, state
SwirldState2
SwirldState2
preHandle
preHandle
transaction
transaction
ConsensusRound, state
ConsensusRound, state
handleConsensusRound
handleConsensusRound
round, dual state
round, dual state
handleRound
handleRound
handleConsensusRound
handleConsensusRound
ConsensusRoundHandler
ConsensusRoundHandler
Consensus Round Queue
(thread-cons)
Consensus Round Queue...
InterruptableConsumer<ConsensusRound>
InterruptableConsumer<ConsensusRound>
signed state
signed state
applyConsensusRoundToState
applyConsensusRoundToState
ConsensusRound
ConsensusRound
ConsensusRound
ConsensusRound
ConsensusRound
ConsensusRound
addConsensusRound
addConsensusRound
EventStreamManager
EventStreamManager
ConsensusRoundObserver
ConsensusRoundObserver
ConsensusRound
ConsensusRound
consensusRound
consensusRound
EventImpl
EventImpl
addEvents
addEvents
EventImpls
EventImpls
addEvent
addEvent
Multistream
Multistream
LinkedObjectStream
LinkedObjectStream
EventImpl
EventImpl
addObject
addObject
EventImpl
EventImpl
Hash Queue Thread
(hash-queue)
Hash Queue Thread...
Write Queue Thread
(write-queue)
Write Queue Thread...
EventImpl
EventImpl
ConsensusRoundObserver
ConsensusRoundObserver
ConsensusRound
ConsensusRound
consensusRound
consensusRound
ConsensusRound
ConsensusRound
handleConsensus
handleConsensus
OrphanBufferingLinker
OrphanBufferingLinker
linkEvent
linkEvent
GossipEvent
GossipEvent
pollLinkedEvent
pollLinkedEvent
calls
calls
updateGenerations
updateGenerations
EventImpl
EventImpl
TODO there are more observers
TODO there a...
ConsensusWrapper
ConsensusWrapper
addEvent
addEvent
EventImpl
EventImpl
EventAddedObserver
EventAddedObserver
eventAdded
eventAdded
ConsensusRounds
ConsensusRounds
InterfaceName
InterfaceName
interfaceMethod
interfaceMethod
classMethod
classMethod
EventTaskCreator
EventTaskCreator
TODO
TODO
handleMethod
handleMethod
QueueElement
QueueElement
The color of a thread should be reflected in the arrows between methods
The color of a thread...
TODO there are more observers
TODO there a...
PostConsensusSystemTransactionManagerImpl
PostConsensusSystemTransactionManagerImpl
PreConsensusSystemTransactionManagerImpl
PreConsensusSystemTransactionManagerImpl
PostConsensusSystemTransactionManager
PostConsensusSystemTransactionManager
handleRound
handleRound
PreConsensusSystemTransactionManager
PreConsensusSystemTransactionManager
handleEvent
handleEvent
EventImpl
EventImpl
ConsensusRound, state
ConsensusRound, state
handleStateSignatureTransactionPreConsensus
handleStateSignatureTransactionPreConsensus
handleStateSignatureTransactionPostConsensus
handleStateSignatureTransactionPostConsensus
StateSignatureTransactions
StateSignatureTransactions
StateSignatureTransactions
StateSignatureTransactions
Node Mindmap
Node Mindmap
getStateForSigning
getStateForSigning
copy
copy
State Hashing Queue
(state-hash-sign)
State Hashing Queue...
NewSignedStateFromTransactionsConsumer
NewSignedStateFromTransactionsConsumer
newSignedStateFromTransactions
newSignedStateFromTransactions
SignedStateHasher
SignedStateHasher
hashState
hashState
submitTransaction
submitTransaction
EventTransactionPool
EventTransactionPool
submitTransaction
submitTransaction
addUnsignedState
addUnsignedState
transmitSignature
transmitSignature
RoundAppliedToStateConsumer
RoundAppliedToStateConsumer
roundAppliedToState
roundAppliedToState
round number
round number
roundCompleted
roundCompleted
HashLogger
HashLogger
logHashes
logHashes
SignedStateFileManager
SignedStateFileManager
StateHasEnoughSignaturesConsumer
StateHasEnoughSignaturesConsumer
stateHasEnoughSignatures
stateHasEnoughSignatures
saveSignedStateToDisk
saveSignedStateToDisk
FreezeManager
FreezeManager
stateHasEnoughSignatures
stateHasEnoughSignatures
EventCreationRule
EventCreationRule
shouldCreateEvent
shouldCreateEvent
TransactionSupplier
TransactionSupplier
getTransactions
getTransactions
StateLacksSignaturesConsumer
StateLacksSignaturesConsumer
stateLacksSignatures
stateLacksSignatures
purgeOldStates
purgeOldStates
file system
file system
State to disk
(signed-state-file-manager)
State to disk...
(lambda)
(lambda)
ManualWiring
ManualWiring
(lambda)
(lambda)
stateToDisk
stateToDisk
stateHashedObserver
stateHashedObserverText is not SVG - cannot display \ No newline at end of file +
PreConsensensusEventWriter
PreConsensensusEventWriter
SwirldsPlatform
SwirldsPlatform
createPrioritySystemTransaction
createPrioritySystemTransaction
SignatureTransmitter
SignatureTransmitter
Template
Template
ClassName
ClassName
ClassName
ClassName
PreConsensusEventHandler
PreConsensusEventHandler
PreConsensusEventObserver
PreConsensusEventObserver
EventObserverDispatcher
EventObserverDispatcher
EventIntake
EventIntake
Consumer<GossipEvent>
Consumer<GossipEvent>
EventValidator
EventValidator
EventTaskDispatcher
EventTaskDispatcher
Queue Thread Name
(thread-name)
Queue Thread Name...
Note: the name in the parenthesis should match the value set with QueueThreadConfiguration::setThreadName
Note: the name in the parenthes...
EventIntakeTask
EventIntakeTask
Event Intake Queue
(event-intake)
Event Intake Queue...
EventCreator
EventCreator
createEvent
createEvent
pathway not used with OOO gossip
pathway not used with OOO gossip
GossipEvent
GossipEvent
EventImpl
EventImpl
addUnlinkedEvent
addUnlinkedEvent
ConsensusRounds
ConsensusRounds
addEvent
addEvent
dispatchTask
dispatchTask
Conditional
Conditional
instanceof CreateEventTask
instanceof CreateEvent...
instanceof ValidEvent
instanceof ValidEvent
instanceof GossipEvent
instanceof GossipEvent
CreateEventTask
CreateEventTask
GossipEvent
GossipEvent
GossipEvent
GossipEvent
validateEvent
validateEvent
PreConsensusEventObserver
PreConsensusEventObserver
preConsensusEvent
preConsensusEvent
EventImpl
EventImpl
EventImpl
EventImpl
preConsensusEvent
preConsensusEvent
PreConsensus Event Queue
(thread-curr)
PreConsensus Event Queue...
SwirldStateManagerImpl
SwirldStateManagerImpl
SwirldStateManager
SwirldStateManager
handlePreConsensusEvent
handlePreConsensusEvent
EventImpl
EventImpl
preHandle
preHandle
EventImpl
EventImpl
DefaultStateManagementComponent
DefaultStateManagementComponent
ConsensusHashManager
ConsensusHashManager
SignedStateManager
SignedStateManager
postConsensusSignatureObserver
postConsensusSignatureObserver
signature info
signature info
preConsensusSignatureObserver
preConsensusSignatureObserver
signature info
signature info
TransactionHandler
TransactionHandler
preHandle
preHandle
EventImpl, state
EventImpl, state
SwirldState2
SwirldState2
preHandle
preHandle
transaction
transaction
ConsensusRound, state
ConsensusRound, state
handleConsensusRound
handleConsensusRound
round, dual state
round, dual state
handleRound
handleRound
handleConsensusRound
handleConsensusRound
ConsensusRoundHandler
ConsensusRoundHandler
Consensus Round Queue
(thread-cons)
Consensus Round Queue...
InterruptableConsumer<ConsensusRound>
InterruptableConsumer<ConsensusRound>
signed state
signed state
applyConsensusRoundToState
applyConsensusRoundToState
ConsensusRound
ConsensusRound
ConsensusRound
ConsensusRound
ConsensusRound
ConsensusRound
addConsensusRound
addConsensusRound
EventStreamManager
EventStreamManager
ConsensusRoundObserver
ConsensusRoundObserver
ConsensusRound
ConsensusRound
consensusRound
consensusRound
EventImpl
EventImpl
addEvents
addEvents
EventImpls
EventImpls
addEvent
addEvent
Multistream
Multistream
LinkedObjectStream
LinkedObjectStream
EventImpl
EventImpl
addObject
addObject
EventImpl
EventImpl
Hash Queue Thread
(hash-queue)
Hash Queue Thread...
Write Queue Thread
(write-queue)
Write Queue Thread...
EventImpl
EventImpl
ConsensusRoundObserver
ConsensusRoundObserver
consensusRound
consensusRound
ConsensusRound
ConsensusRound
handleConsensus
handleConsensus
OrphanBufferingLinker
OrphanBufferingLinker
linkEvent
linkEvent
GossipEvent
GossipEvent
pollLinkedEvent
pollLinkedEvent
calls
calls
updateGenerations
updateGenerations
EventImpl
EventImpl
TODO there are more observers
TODO there a...
ConsensusWrapper
ConsensusWrapper
addEvent
addEvent
EventImpl
EventImpl
EventAddedObserver
EventAddedObserver
eventAdded
eventAdded
ConsensusRounds
ConsensusRounds
InterfaceName
InterfaceName
interfaceMethod
interfaceMethod
classMethod
classMethod
EventTaskCreator
EventTaskCreator
TODO
TODO
handleMethod
handleMethod
QueueElement
QueueElement
The color of a thread should be reflected in the arrows between methods
The color of a thread...
TODO there are more observers
TODO there a...
PostConsensusSystemTransactionManagerImpl
PostConsensusSystemTransactionManagerImpl
PreConsensusSystemTransactionManagerImpl
PreConsensusSystemTransactionManagerImpl
PostConsensusSystemTransactionManager
PostConsensusSystemTransactionManager
handleRound
handleRound
PreConsensusSystemTransactionManager
PreConsensusSystemTransactionManager
handleEvent
handleEvent
EventImpl
EventImpl
ConsensusRound, state
ConsensusRound, state
handleStateSignatureTransactionPreConsensus
handleStateSignatureTransactionPreConsensus
handleStateSignatureTransactionPostConsensus
handleStateSignatureTransactionPostConsensus
StateSignatureTransactions
StateSignatureTransactions
StateSignatureTransactions
StateSignatureTransactions
Node Mindmap
Node Mindmap
getStateForSigning
getStateForSigning
copy
copy
State Hashing Queue
(state-hash-sign)
State Hashing Queue...
NewSignedStateFromTransactionsConsumer
NewSignedStateFromTransactionsConsumer
newSignedStateFromTransactions
newSignedStateFromTransactions
SignedStateHasher
SignedStateHasher
hashState
hashState
submitTransaction
submitTransaction
EventTransactionPool
EventTransactionPool
submitTransaction
submitTransaction
addUnsignedState
addUnsignedState
transmitSignature
transmitSignature
RoundAppliedToStateConsumer
RoundAppliedToStateConsumer
roundAppliedToState
roundAppliedToState
round number
round number
roundCompleted
roundCompleted
HashLogger
HashLogger
logHashes
logHashes
SignedStateFileManager
SignedStateFileManager
StateHasEnoughSignaturesConsumer
StateHasEnoughSignaturesConsumer
stateHasEnoughSignatures
stateHasEnoughSignatures
saveSignedStateToDisk
saveSignedStateToDisk
FreezeManager
FreezeManager
stateHasEnoughSignatures
stateHasEnoughSignatures
EventCreationRule
EventCreationRule
shouldCreateEvent
shouldCreateEvent
TransactionSupplier
TransactionSupplier
getTransactions
getTransactions
StateLacksSignaturesConsumer
StateLacksSignaturesConsumer
stateLacksSignatures
stateLacksSignatures
purgeOldStates
purgeOldStates
file system
file system
State to disk
(signed-state-file-manager)
State to disk...
(lambda)
(lambda)
ManualWiring
ManualWiring
(lambda)
(lambda)
stateToDisk
stateToDisk
stateHashedObserver
stateHashedObserver
PreConsensus Event Writer Queue
PreConsensus Event Writer Queue
writeEvent
writeEvent
setMinimumGenerationNonAncient
setMinimumGenerationNonAncient
setMinimumGenerationToStore
setMinimumGenerationToStore
requestFlush
requestFlush
waitUntilDurable
waitUntilDurable
write events to disk
write events to di...Text is not SVG - cannot display \ No newline at end of file diff --git a/platform-sdk/docs/threads/eventflow-threads.html b/platform-sdk/docs/threads/eventflow-threads.html index 085d92dbb52d..e9adbbe8990a 100644 --- a/platform-sdk/docs/threads/eventflow-threads.html +++ b/platform-sdk/docs/threads/eventflow-threads.html @@ -5,7 +5,7 @@ eventflow-threads -
+
\ No newline at end of file diff --git a/platform-sdk/platform-apps/tests/AddressBookTestingTool/README.md b/platform-sdk/platform-apps/tests/AddressBookTestingTool/README.md new file mode 100644 index 000000000000..134577ec7777 --- /dev/null +++ b/platform-sdk/platform-apps/tests/AddressBookTestingTool/README.md @@ -0,0 +1,179 @@ +# AddressBookTestingTool Instructions + +This document describes various manual tests that were performed using this testing app. This playbook should be used when designing automated tests using this app. + +## Global Modifications and Starting Conditions + +Be sure to clean and assemble the whole project if the java files have been modified. + +### config.txt +comment out the current app +``` +# app, HashgraphDemo.jar, 1,0,0,0,0,0,0,0,0,0, all +``` +uncomment the AddressBookTestingTool.jar +``` + app, AddressBookTestingTool.jar, +``` + +### AddressTestingToolMain.java + +If following these instructions sequentially, ensure the software version is 1. + +```java +private static BasicSoftwareVersion softwareVersion = new BasicSoftwareVersion(1); +``` + +### AddressTestingToolState.java + +If following these instructions sequentially, ensure the staking profile is 1. + +```java +private int stakingProfile = 1; +``` + + + +## Testing Genesis Behavior + +### Force Use of Config Address Book on Genesis +#### Instructions +1. Delete `sdk/data/saved` directory if it exists +2. Ensure settings.txt has `state.saveStatePeriod, 0` +3. Add to settings.txt the value `state.forceUseOfConfigAddressBook, true` +4. Run the app for a short time. + +#### Validation + +* check the swirlds.log for the text + +``` +AddressBookInitializer: A setting has forced the use of the configuration address. +``` + +* check the directory `sdk/data/saved/address_book` for files + * usedAddressBook_v1_.txt + * matches the addresses in the config.txt, including stake value. + * usedAddressBook_v1_.txt.debug + * The configuration address book is the same as what is in config.txt + * the state saved address book was null + * the used address book text says `The Configuration Address Book Was Used.` + + +### Call to SwirldState.updateStake() on Genesis +#### Instructions +1. Delete `sdk/data/saved` directory if it exists +2. **Ensure settings.txt has `state.saveStatePeriod, 10` (differs from previous section)** +3. **Ensure settings.txt has the value `state.forceUseOfConfigAddressBook, false` (differs from previous section)** +4. **Run the app for 20 seconds. (differs from previous section)** + +#### Validation + +* check the swirlds.log for the text + +``` +AddressBookInitializer: The loaded signed state is null. The candidateAddressBook is set to genesisSwirldState.updateStake(configAddressBook, null). +``` + +* check the directory `sdk/data/saved/address_book` for files + * usedAddressBook_v1_.txt + * **contains the addresses in the config.txt, all with stake 10. (differs from previous section)** + * usedAddressBook_v1_.txt.debug + * The configuration address book is the same as what is in config.txt + * the state saved address book was null + * **the used address book matches the content of the non-debug .txt file. (differs from previous section)** + +## Testing Non-Genesis Behavior, No Software Upgrade + +### No Software Upgrade, Use Saved State Address Book +#### Instructions +1. Ensure settings.txt has `state.saveStatePeriod, 10` +2. Ensure settings.txt has the value `state.forceUseOfConfigAddressBook, false` +3. Run the app for 20 seconds. + +#### Validation + +* check the swirlds.log for the text + +``` +AddressBookInitializer: No Software Upgrade. Continuing with software version 1 and using the loaded signed state's address book and stake values. +``` + +* check the directory `sdk/data/saved/address_book` for the latest files + * usedAddressBook_v1_.txt + * contains the addresses in the config.txt, all with stake 10. + * usedAddressBook_v1_.txt.debug + * The configuration address book is the same as what is in config.txt + * **the state saved address book matches the content of the non-debug .txt file. (differs from previous section)** + * **the used address book has the text `The State Saved Address Book Was Used.` (differs from previous section)** + +### No Software Upgrade, Force Use of Config Address Book +#### Instructions +1. Ensure settings.txt has `state.saveStatePeriod, 10` +2. **Ensure settings.txt has the value `state.forceUseOfConfigAddressBook, true` (differs from previous section)** +3. Run the app for 20 seconds. + +#### Validation + +* check the swirlds.log for the text + +``` +AddressBookInitializer: A setting has forced the use of the configuration address. +``` + +* check the directory `sdk/data/saved/address_book` for the latest files + * usedAddressBook_v1_.txt + * **matches the addresses in the config.txt, including stake value. (differs from previous section)** + * usedAddressBook_v1_.txt.debug + * The configuration address book is the same as what is in config.txt + * **the state saved address book contains the addresses in the config.txt, all with stake 10. (differs from previous section)** + * **the used address book has the text `The Configuration Address Book Was Used.` (differs from previous section)** + +### No Software Upgrade, Use Saved State Address Book, Matches Config +#### Instructions +1. Ensure settings.txt has `state.saveStatePeriod, 10` +2. **Ensure settings.txt has the value `state.forceUseOfConfigAddressBook, false` (differs from previous section)** +3. Run the app for 20 seconds. + +#### Validation + +* check the swirlds.log for the text + +``` +AddressBookInitializer: No Software Upgrade. Continuing with software version 1 and using the loaded signed state's address book and stake values. +``` + +* check the directory `sdk/data/saved/address_book` for the latest files + * usedAddressBook_v1_.txt + * matches the addresses in the config.txt, including stake value. + * usedAddressBook_v1_.txt.debug + * The configuration address book is the same as what is in config.txt + * **The state saved address book is the same as what is in the config.txt (differs from previous section)** + * **the used address book has the text `The State Saved Address Book Was Used.` (differs from previous section)** + +## Testing Non-Genesis Behavior, Software Upgrade + +### Software Upgrade, Force Use of Config Address Book +#### Instructions +1. Ensure settings.txt has `state.saveStatePeriod, 10` +2. Ensure settings.txt has the value `state.forceUseOfConfigAddressBook, false` +3. **Increase the softwareVersion in AddressBookTestingToolMain.java to 2. (differs from previous section)** +4. **Change the stakingProfile in AddressBookTestingToolState.java to 2. (differs from previous section)** +5. **recompile the application: assemble ONLY!!!!.(differs from previous section)** +6. Run the app for 20 seconds. + +#### Validation + +* check the swirlds.log for the text + +``` +AddressBookInitializer: Software Upgrade from version 1 to 2. The address book stake will be updated by the saved state's SwirldState. +``` + +* check the directory `sdk/data/saved/address_book` for the latest files + * usedAddressBook_v1_.txt + * **matches the addresses in the config.txt, but the stake values incrementally increase starting from 0. (differs from previous section)** + * usedAddressBook_v1_.txt.debug + * The configuration address book is the same as what is in config.txt + * The state saved address book is the same as what is in config.txt + * **the used address book matches the content of the non-debug .txt file. (differs from previous section)** diff --git a/platform-sdk/platform-apps/tests/AddressBookTestingTool/build.gradle.kts b/platform-sdk/platform-apps/tests/AddressBookTestingTool/build.gradle.kts new file mode 100644 index 000000000000..42b6aec135bd --- /dev/null +++ b/platform-sdk/platform-apps/tests/AddressBookTestingTool/build.gradle.kts @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2020-2023 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id("com.swirlds.platform.conventions") + id("com.swirlds.platform.application") +} + +dependencies { + // Individual Dependencies + implementation(project(":swirlds-platform-core")) + implementation(libs.bundles.logging.impl) + compileOnly(libs.spotbugs.annotations) +} diff --git a/platform-sdk/platform-apps/tests/AddressBookTestingTool/gradle.properties b/platform-sdk/platform-apps/tests/AddressBookTestingTool/gradle.properties new file mode 100644 index 000000000000..9b4e94c7647a --- /dev/null +++ b/platform-sdk/platform-apps/tests/AddressBookTestingTool/gradle.properties @@ -0,0 +1,18 @@ +# +# Copyright 2016-2022 Hedera Hashgraph, LLC +# +# This software is the confidential and proprietary information of +# Hedera Hashgraph, LLC. ("Confidential Information"). You shall not +# disclose such Confidential Information and shall use it only in +# accordance with the terms of the license agreement you entered into +# with Hedera Hashgraph. +# +# HEDERA HASHGRAPH MAKES NO REPRESENTATIONS OR WARRANTIES ABOUT THE SUITABILITY OF +# THE SOFTWARE, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +# TO THE IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +# PARTICULAR PURPOSE, OR NON-INFRINGEMENT. HEDERA HASHGRAPH SHALL NOT BE LIABLE FOR +# ANY DAMAGES SUFFERED BY LICENSEE AS A RESULT OF USING, MODIFYING OR +# DISTRIBUTING THIS SOFTWARE OR ITS DERIVATIVES. +# + +mainClass=com.swirlds.demo.addressbook.AddressBookTestingToolMain diff --git a/platform-sdk/platform-apps/tests/AddressBookTestingTool/src/main/java/com/swirlds/demo/addressbook/AddressBookTestingToolMain.java b/platform-sdk/platform-apps/tests/AddressBookTestingTool/src/main/java/com/swirlds/demo/addressbook/AddressBookTestingToolMain.java new file mode 100644 index 000000000000..27dc439664e0 --- /dev/null +++ b/platform-sdk/platform-apps/tests/AddressBookTestingTool/src/main/java/com/swirlds/demo/addressbook/AddressBookTestingToolMain.java @@ -0,0 +1,116 @@ +/* + * Copyright (C) 2022-2023 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.swirlds.demo.addressbook; + +import static com.swirlds.logging.LogMarker.STARTUP; + +import com.swirlds.common.system.BasicSoftwareVersion; +import com.swirlds.common.system.NodeId; +import com.swirlds.common.system.Platform; +import com.swirlds.common.system.PlatformWithDeprecatedMethods; +import com.swirlds.common.system.SwirldMain; +import com.swirlds.common.system.SwirldState; +import edu.umd.cs.findbugs.annotations.NonNull; +import java.security.SecureRandom; +import java.util.Objects; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +/** + *

+ * An application that updates address book stake weights on version upgrade. + *

+ * + *

+ * Arguments: + *

    + *
  1. + * No arguments parsed at this time. The software version must be updated through setting the static value in + * this main class and recompiling. The behavior of staking is updated in the State class and recompiling. + *
    2. + *
+ */ +public class AddressBookTestingToolMain implements SwirldMain { + /** The logger for this class. */ + private static final Logger logger = LogManager.getLogger(AddressBookTestingToolMain.class); + + /** The default software version of this application. */ + private static final BasicSoftwareVersion softwareVersion = new BasicSoftwareVersion(1); + + /** The platform. */ + private Platform platform; + + /** The number of transactions to generate per second. */ + private static final int TRANSACTIONS_PER_SECOND = 100; + + public AddressBookTestingToolMain() { + logger.info(STARTUP.getMarker(), "constructor called in Main."); + } + + /** + * {@inheritDoc} + */ + @Override + public void init(@NonNull final Platform platform, @NonNull final NodeId id) { + Objects.requireNonNull(platform, "The platform must not be null."); + Objects.requireNonNull(id, "The node id must not be null."); + + logger.info(STARTUP.getMarker(), "init called in Main for node {}.", id); + this.platform = platform; + + parseArguments(((PlatformWithDeprecatedMethods) platform).getParameters()); + } + + /** + * Parses the arguments. Currently, no arguments are expected. + * + * @param args the arguments + * @throws IllegalArgumentException if the arguments array has length other than 0. + */ + private void parseArguments(@NonNull final String[] args) { + Objects.requireNonNull(args, "The arguments must not be null."); + if (args.length != 0) { + throw new IllegalArgumentException("Expected no arguments. See javadocs for details."); + } + } + + /** + * {@inheritDoc} + */ + @Override + public void run() { + logger.info(STARTUP.getMarker(), "run called in Main."); + new TransactionGenerator(new SecureRandom(), platform, TRANSACTIONS_PER_SECOND).start(); + } + + /** + * {@inheritDoc} + */ + @Override + public SwirldState newState() { + return new AddressBookTestingToolState(); + } + + /** + * {@inheritDoc} + */ + @Override + public BasicSoftwareVersion getSoftwareVersion() { + logger.info(STARTUP.getMarker(), "returning software version {}", softwareVersion); + return softwareVersion; + } +} diff --git a/platform-sdk/platform-apps/tests/AddressBookTestingTool/src/main/java/com/swirlds/demo/addressbook/AddressBookTestingToolState.java b/platform-sdk/platform-apps/tests/AddressBookTestingTool/src/main/java/com/swirlds/demo/addressbook/AddressBookTestingToolState.java new file mode 100644 index 000000000000..db18294f0141 --- /dev/null +++ b/platform-sdk/platform-apps/tests/AddressBookTestingTool/src/main/java/com/swirlds/demo/addressbook/AddressBookTestingToolState.java @@ -0,0 +1,241 @@ +/* + * Copyright (C) 2022-2023 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.swirlds.demo.addressbook; +/* + * This file is public domain. + * + * SWIRLDS MAKES NO REPRESENTATIONS OR WARRANTIES ABOUT THE SUITABILITY OF + * THE SOFTWARE, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED + * TO THE IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A + * PARTICULAR PURPOSE, OR NON-INFRINGEMENT. SWIRLDS SHALL NOT BE LIABLE FOR + * ANY DAMAGES SUFFERED AS A RESULT OF USING, MODIFYING OR + * DISTRIBUTING THIS SOFTWARE OR ITS DERIVATIVES. + */ + +import static com.swirlds.logging.LogMarker.STARTUP; + +import com.swirlds.common.io.streams.SerializableDataInputStream; +import com.swirlds.common.io.streams.SerializableDataOutputStream; +import com.swirlds.common.merkle.MerkleLeaf; +import com.swirlds.common.merkle.impl.PartialMerkleLeaf; +import com.swirlds.common.system.InitTrigger; +import com.swirlds.common.system.Platform; +import com.swirlds.common.system.PlatformWithDeprecatedMethods; +import com.swirlds.common.system.Round; +import com.swirlds.common.system.SoftwareVersion; +import com.swirlds.common.system.SwirldDualState; +import com.swirlds.common.system.SwirldState; +import com.swirlds.common.system.address.AddressBook; +import com.swirlds.common.system.events.ConsensusEvent; +import com.swirlds.common.system.transaction.ConsensusTransaction; +import com.swirlds.common.utility.ByteUtils; +import edu.umd.cs.findbugs.annotations.NonNull; +import edu.umd.cs.findbugs.annotations.Nullable; +import java.io.IOException; +import java.util.Iterator; +import java.util.Objects; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +/** + * State for the AddressBookTestingTool. + */ +public class AddressBookTestingToolState extends PartialMerkleLeaf implements SwirldState, MerkleLeaf { + + private static final Logger logger = LogManager.getLogger(AddressBookTestingToolState.class); + + /** modify this value to change how updateStake behaves. */ + private static final int STAKING_PROFILE = 1; + + private static class ClassVersion { + public static final int ORIGINAL = 1; + } + + private static final long CLASS_ID = 0xf052378c7364ef47L; + + private long selfId; + + /** + * The true "state" of this app. Each transaction is just an integer that gets added to this value. + */ + private long runningSum = 0; + + public AddressBookTestingToolState() { + logger.info(STARTUP.getMarker(), "New State Constructed."); + } + + /** + * Copy constructor. + */ + private AddressBookTestingToolState(@NonNull final AddressBookTestingToolState that) { + super(that); + Objects.requireNonNull(that, "the address book testing tool state to copy cannot be null"); + this.runningSum = that.runningSum; + this.selfId = that.selfId; + } + + /** + * {@inheritDoc} + */ + @Override + public synchronized AddressBookTestingToolState copy() { + throwIfImmutable(); + return new AddressBookTestingToolState(this); + } + + /** + * {@inheritDoc} + */ + @Override + public void init( + @NonNull final Platform platform, + @NonNull final SwirldDualState swirldDualState, + @NonNull final InitTrigger trigger, + @Nullable final SoftwareVersion previousSoftwareVersion) { + Objects.requireNonNull(platform, "the platform cannot be null"); + Objects.requireNonNull(swirldDualState, "the swirld dual state cannot be null"); + Objects.requireNonNull(trigger, "the init trigger cannot be null"); + + logger.info(STARTUP.getMarker(), "init called in State."); + throwIfImmutable(); + + if (trigger == InitTrigger.GENESIS) { + parseArguments(((PlatformWithDeprecatedMethods) platform).getParameters()); + } + + this.selfId = platform.getSelfId().getId(); + } + + /** + * {@inheritDoc} + */ + @Override + public void handleConsensusRound(@NonNull final Round round, @NonNull final SwirldDualState swirldDualState) { + Objects.requireNonNull(round, "the round cannot be null"); + Objects.requireNonNull(swirldDualState, "the swirld dual state cannot be null"); + throwIfImmutable(); + + final Iterator eventIterator = round.iterator(); + + while (eventIterator.hasNext()) { + final ConsensusEvent event = eventIterator.next(); + event.consensusTransactionIterator().forEachRemaining(this::handleTransaction); + } + } + + /** + * Apply a transaction to the state. + * + * @param transaction the transaction to apply + */ + private void handleTransaction(@NonNull final ConsensusTransaction transaction) { + final int delta = ByteUtils.byteArrayToInt(transaction.getContents(), 0); + runningSum += delta; + } + + /** + * {@inheritDoc} + */ + @Override + public void serialize(@NonNull final SerializableDataOutputStream out) throws IOException { + Objects.requireNonNull(out, "the serializable data output stream cannot be null"); + out.writeLong(runningSum); + } + + /** + * {@inheritDoc} + */ + @Override + public void deserialize(@NonNull final SerializableDataInputStream in, final int version) throws IOException { + Objects.requireNonNull(in, "the serializable data input stream cannot be null"); + runningSum = in.readLong(); + } + + /** + * {@inheritDoc} + */ + private void parseArguments(@NonNull final String[] args) { + if (args.length != 0) { + throw new IllegalArgumentException("Expected no arguments. See javadocs for details."); + } + } + + /** + * {@inheritDoc} + */ + @Override + public long getClassId() { + return CLASS_ID; + } + + /** + * {@inheritDoc} + */ + @Override + public int getVersion() { + return ClassVersion.ORIGINAL; + } + + /** + * {@inheritDoc} + */ + @Override + @NonNull + public AddressBook updateStake(@NonNull final AddressBook addressBook) { + Objects.requireNonNull(addressBook, "the address book cannot be null"); + logger.info("updateStake called in State. Staking Profile: {}", STAKING_PROFILE); + switch (STAKING_PROFILE) { + case 1: + return stakingProfile1(addressBook); + case 2: + return stakingProfile2(addressBook); + default: + logger.info(STARTUP.getMarker(), "Staking Profile {}: no change to address book.", STAKING_PROFILE); + return addressBook; + } + } + + /** + * All nodes received 10 stake. + * + * @param addressBook the address book to update. + * @return the updated address book. + */ + @NonNull + private AddressBook stakingProfile1(@NonNull final AddressBook addressBook) { + logger.info(STARTUP.getMarker(), "Staking Profile 1: updating all nodes to have 10 stake."); + for (int i = 0; i < addressBook.getSize(); i++) { + addressBook.updateStake(i, 10); + } + return addressBook; + } + + /** + * All nodes received stake equal to their nodeId. + * + * @param addressBook the address book to update. + * @return the updated address book. + */ + @NonNull + private AddressBook stakingProfile2(@NonNull final AddressBook addressBook) { + logger.info(STARTUP.getMarker(), "Staking Profile 2: updating all nodes to have stake equal to their nodeId."); + for (int i = 0; i < addressBook.getSize(); i++) { + addressBook.updateStake(i, i); + } + return addressBook; + } +} diff --git a/platform-sdk/platform-apps/tests/AddressBookTestingTool/src/main/java/com/swirlds/demo/addressbook/TransactionGenerator.java b/platform-sdk/platform-apps/tests/AddressBookTestingTool/src/main/java/com/swirlds/demo/addressbook/TransactionGenerator.java new file mode 100644 index 000000000000..a87b6a62bc49 --- /dev/null +++ b/platform-sdk/platform-apps/tests/AddressBookTestingTool/src/main/java/com/swirlds/demo/addressbook/TransactionGenerator.java @@ -0,0 +1,76 @@ +/* + * Copyright (C) 2022-2023 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.swirlds.demo.addressbook; + +import static com.swirlds.common.threading.manager.AdHocThreadManager.getStaticThreadManager; +import static com.swirlds.common.utility.ByteUtils.intToByteArray; + +import com.swirlds.common.system.Platform; +import com.swirlds.common.threading.framework.StoppableThread; +import com.swirlds.common.threading.framework.config.StoppableThreadConfiguration; +import com.swirlds.common.utility.Startable; +import edu.umd.cs.findbugs.annotations.NonNull; +import java.util.Objects; +import java.util.Random; + +/** + * Generates and submits transactional workload. + */ +public class TransactionGenerator implements Startable { + + private final Random random; + private final Platform platform; + + private final StoppableThread thread; + + public TransactionGenerator( + @NonNull final Random random, + @NonNull final Platform platform, + final int networkWideTransactionsPerSecond) { + Objects.requireNonNull(random, "The random number generator must not be null"); + Objects.requireNonNull(platform, "The platform must not be null"); + this.random = random; + this.platform = platform; + + // Each node in an N node network should create 1/N transactions per second. + final int tps = + networkWideTransactionsPerSecond / platform.getAddressBook().getSize(); + + thread = new StoppableThreadConfiguration<>(getStaticThreadManager()) + .setComponent("addressbook-testing-tool") + .setThreadName("transaction-generator") + .setMaximumRate(tps) + .setWork(this::generateTransaction) + .build(); + } + + /** + * {@inheritDoc} + */ + @Override + public void start() { + thread.start(); + } + + /** + * Generate and submit a single transaction. + */ + private void generateTransaction() { + // Transactions are simple: take an integer, and add it into the running sum. + platform.createTransaction(intToByteArray(random.nextInt())); + } +} diff --git a/platform-sdk/platform-apps/tests/AddressBookTestingTool/src/main/java/module-info.java b/platform-sdk/platform-apps/tests/AddressBookTestingTool/src/main/java/module-info.java new file mode 100644 index 000000000000..0945fbfdff87 --- /dev/null +++ b/platform-sdk/platform-apps/tests/AddressBookTestingTool/src/main/java/module-info.java @@ -0,0 +1,10 @@ +module com.swirlds.demo.addressbook { + requires com.swirlds.platform; + requires com.swirlds.common; + requires com.swirlds.logging; + requires lazysodium.java; + requires org.bouncycastle.provider; + requires org.bouncycastle.pkix; + requires org.apache.logging.log4j; + requires static com.github.spotbugs.annotations; +} diff --git a/platform-sdk/sdk/config.txt b/platform-sdk/sdk/config.txt index 32096939b920..fe105c9bc569 100644 --- a/platform-sdk/sdk/config.txt +++ b/platform-sdk/sdk/config.txt @@ -17,7 +17,8 @@ swirld, 123 # app, FCMStatsTestingTool.jar, 0, 0, -1, 250 # app, PlatformTestingTool.jar, FCMFCQ-Basic-1k-10m.json # app, MigrationTestingTool.jar, 4685095852903873060, 5000, 4 -# app, ISSTestingTool.jar, 10000, 60:0-1+2+3, 120:0-1-2+3 +# app, ISSTestingTool.jar, 10000, 60:0-1+2+3, 120:0-1-2+3 +# app, AddressBookTestingTool.jar, # ** END REMOVE FROM SDK RELEASES ** address, A, Alice, 1, 127.0.0.1, 15301, 127.0.0.1, 15301 diff --git a/platform-sdk/settings.gradle.kts b/platform-sdk/settings.gradle.kts index b41b70b9ac8e..778b31d92917 100644 --- a/platform-sdk/settings.gradle.kts +++ b/platform-sdk/settings.gradle.kts @@ -58,6 +58,8 @@ include(":swirlds-platform-apps:demos:HelloSwirldDemo") include(":swirlds-platform-apps:demos:StatsDemo") +include(":swirlds-platform-apps:tests:AddressBookTestingTool") + include(":swirlds-platform-apps:tests:ISSTestingTool") include(":swirlds-platform-apps:tests:MigrationTestingTool") diff --git a/platform-sdk/swirlds-common/src/main/java/com/swirlds/common/config/StateConfig.java b/platform-sdk/swirlds-common/src/main/java/com/swirlds/common/config/StateConfig.java index cad6da080b44..0cbf65391059 100644 --- a/platform-sdk/swirlds-common/src/main/java/com/swirlds/common/config/StateConfig.java +++ b/platform-sdk/swirlds-common/src/main/java/com/swirlds/common/config/StateConfig.java @@ -92,9 +92,6 @@ * If true, then enable extra debug code that tracks signed states. Very useful for debugging state leaks. * This debug code is relatively expensive (it takes and stores stack traces when operations are * performed on signed state objects). - * @param forceUseOfConfigAddressBook - * If true, then the address book from the config file will be used instead of the address book from the - * signed state and the swirld state will not be queried for any address book updates. */ @ConfigData("state") public record StateConfig( @@ -117,8 +114,7 @@ public record StateConfig( @ConfigProperty(defaultValue = "5") int debugHashDepth, @ConfigProperty(defaultValue = "1000") int maxAgeOfFutureStateSignatures, @ConfigProperty(defaultValue = "26") int roundsToKeepForSigning, - @ConfigProperty(defaultValue = "false") boolean signedStateSentinelEnabled, - @ConfigProperty(defaultValue = "true") boolean forceUseOfConfigAddressBook) { + @ConfigProperty(defaultValue = "false") boolean signedStateSentinelEnabled) { /** * Get the main class name that should be used for signed states. diff --git a/platform-sdk/swirlds-common/src/main/java/com/swirlds/common/system/SwirldMain.java b/platform-sdk/swirlds-common/src/main/java/com/swirlds/common/system/SwirldMain.java index 0adc4b915049..a556b2faf53e 100644 --- a/platform-sdk/swirlds-common/src/main/java/com/swirlds/common/system/SwirldMain.java +++ b/platform-sdk/swirlds-common/src/main/java/com/swirlds/common/system/SwirldMain.java @@ -17,6 +17,8 @@ package com.swirlds.common.system; import com.swirlds.config.api.Configuration; +import com.swirlds.config.api.ConfigurationBuilder; +import edu.umd.cs.findbugs.annotations.NonNull; /** * To implement a swirld, create a class that implements SwirldMain. Its constructor should have no @@ -30,7 +32,17 @@ public interface SwirldMain extends Runnable { * @param configuration * configuration for this node */ - default void setConfiguration(final Configuration configuration) { + default void setConfiguration(@NonNull final Configuration configuration) { + // override if needed + } + + /** + * Update the configuration builder with any additional configuration that is required by this swirld. + * + * @param configurationBuilder + * the configuration builder to update + */ + default void updateConfigurationBuilder(@NonNull final ConfigurationBuilder configurationBuilder) { // override if needed } @@ -50,7 +62,7 @@ default void setConfiguration(final Configuration configuration) { * @param selfId * the ID number for this member (myself) */ - void init(Platform platform, NodeId selfId); + void init(@NonNull final Platform platform, @NonNull final NodeId selfId); /** * This is where the app manages the screen and I/O, and creates transactions as needed. It should diff --git a/platform-sdk/swirlds-common/src/main/java/com/swirlds/common/system/address/Address.java b/platform-sdk/swirlds-common/src/main/java/com/swirlds/common/system/address/Address.java index 7cd9bfb895ee..66acdac3e9fc 100644 --- a/platform-sdk/swirlds-common/src/main/java/com/swirlds/common/system/address/Address.java +++ b/platform-sdk/swirlds-common/src/main/java/com/swirlds/common/system/address/Address.java @@ -842,9 +842,15 @@ public boolean equalsWithoutStake(@NonNull final Address address) { && Arrays.equals(addressExternalIpv4, address.addressExternalIpv4) && Arrays.equals(addressInternalIpv6, address.addressInternalIpv6) && Arrays.equals(addressExternalIpv6, address.addressExternalIpv6) - && Objects.equals(sigPublicKey, address.sigPublicKey) - && Objects.equals(encPublicKey, address.encPublicKey) - && Objects.equals(agreePublicKey, address.agreePublicKey) + && Arrays.equals( + sigPublicKey.getPublicKey().getEncoded(), + address.sigPublicKey.getPublicKey().getEncoded()) + && Arrays.equals( + encPublicKey.getPublicKey().getEncoded(), + address.encPublicKey.getPublicKey().getEncoded()) + && Arrays.equals( + agreePublicKey.getPublicKey().getEncoded(), + address.agreePublicKey.getPublicKey().getEncoded()) && Objects.equals(memo, address.memo); } diff --git a/platform-sdk/swirlds-jasperdb/src/main/java/com/swirlds/merkledb/files/hashmap/Bucket.java b/platform-sdk/swirlds-jasperdb/src/main/java/com/swirlds/merkledb/files/hashmap/Bucket.java index d12cbfa52253..96b58fa6e1d6 100644 --- a/platform-sdk/swirlds-jasperdb/src/main/java/com/swirlds/merkledb/files/hashmap/Bucket.java +++ b/platform-sdk/swirlds-jasperdb/src/main/java/com/swirlds/merkledb/files/hashmap/Bucket.java @@ -213,7 +213,8 @@ public long findValue(final int keyHashCode, final K key, final long notFoundVal * @param value the entry value, this can also be special * HalfDiskHashMap.SPECIAL_DELETE_ME_VALUE to mean delete */ - public void putValue(final int keyHashCode, final K key, final long value) { + public void putValue(final K key, final long value) { + final int keyHashCode = key.hashCode(); try { // scan over all existing key/value entries and see if there is already one for this // key. If there is then update it, otherwise we have at least worked out the entryOffset diff --git a/platform-sdk/swirlds-jasperdb/src/main/java/com/swirlds/merkledb/files/hashmap/HalfDiskHashMap.java b/platform-sdk/swirlds-jasperdb/src/main/java/com/swirlds/merkledb/files/hashmap/HalfDiskHashMap.java index c121e0cfef10..26b7f78617bc 100644 --- a/platform-sdk/swirlds-jasperdb/src/main/java/com/swirlds/merkledb/files/hashmap/HalfDiskHashMap.java +++ b/platform-sdk/swirlds-jasperdb/src/main/java/com/swirlds/merkledb/files/hashmap/HalfDiskHashMap.java @@ -18,6 +18,7 @@ import static com.swirlds.logging.LogMarker.EXCEPTION; import static com.swirlds.logging.LogMarker.MERKLE_DB; +import static com.swirlds.merkledb.files.DataFileCommon.NON_EXISTENT_DATA_LOCATION; import static com.swirlds.merkledb.files.DataFileCommon.formatSizeBytes; import static com.swirlds.merkledb.files.DataFileCommon.getSizeOfFiles; import static com.swirlds.merkledb.files.DataFileCommon.logMergeStats; @@ -45,7 +46,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.eclipse.collections.api.tuple.primitive.IntObjectPair; -import org.eclipse.collections.impl.list.mutable.primitive.LongArrayList; import org.eclipse.collections.impl.map.mutable.primitive.IntObjectHashMap; /** @@ -69,8 +69,6 @@ public class HalfDiskHashMap> implements AutoClo private static final String METADATA_FILENAME_SUFFIX = "_metadata.hdhm"; /** Bucket index file name suffix with extension */ private static final String BUCKET_INDEX_FILENAME_SUFFIX = "_bucket_index.ll"; - /** Each index change includes both a bucket index and bucket location. */ - private static final int INDEX_CHANGE_COMPONENTS = 2; /** Nominal value for value to say please delete from map. */ protected static final long SPECIAL_DELETE_ME_VALUE = Long.MIN_VALUE; /** The amount of data used for storing key hash code */ @@ -416,7 +414,6 @@ public void endWriting() throws IOException { // write to files fileCollection.startWriting(); // for each changed bucket, write the new buckets to file but do not update index yet - final LongArrayList indexChanges = new LongArrayList(); int oldBucketIndex = -1; for (IntObjectPair> keyValue : oneTransactionsData.keyValuesView().toList().sortThis()) { @@ -438,13 +435,16 @@ public void endWriting() throws IOException { } final Bucket finalBucket = bucket; // for each changed key in bucket, update bucket - bucketMap.forEachKeyValue((k, v) -> finalBucket.putValue(k.hashCode(), k, v)); - // save bucket - final long bucketLocation = fileCollection.storeDataItem(bucket); - - // stash update bucketIndexToBucketLocation - indexChanges.add(bucketIndex); - indexChanges.add(bucketLocation); + bucketMap.forEachKeyValue(finalBucket::putValue); + if (finalBucket.getSize() == 0) { + // bucket is missing or empty, remove it from the index + bucketIndexToBucketLocation.put(bucketIndex, NON_EXISTENT_DATA_LOCATION); + } else { + // save bucket + final long bucketLocation = fileCollection.storeDataItem(bucket); + // update bucketIndexToBucketLocation + bucketIndexToBucketLocation.put(bucketIndex, bucketLocation); + } } catch (IllegalStateException e) { printStats(); debugDumpTransactionCacheCondensed(); @@ -454,13 +454,6 @@ public void endWriting() throws IOException { } // close files session final DataFileReader> dataFileReader = fileCollection.endWriting(0, numOfBuckets); - // for each changed bucket update index - for (int i = 0; i < indexChanges.size(); i += INDEX_CHANGE_COMPONENTS) { - final long bucketIndex = indexChanges.get(i); - final long bucketLocation = indexChanges.get(i + 1); - // update bucketIndexToBucketLocation - bucketIndexToBucketLocation.put(bucketIndex, bucketLocation); - } // we have updated all indexes so the data file can now be included in merges dataFileReader.setFileCompleted(); } diff --git a/platform-sdk/swirlds-jasperdb/src/test/java/com/swirlds/merkledb/files/hashmap/BucketTest.java b/platform-sdk/swirlds-jasperdb/src/test/java/com/swirlds/merkledb/files/hashmap/BucketTest.java index 30bc091d74b3..495585d0cffc 100644 --- a/platform-sdk/swirlds-jasperdb/src/test/java/com/swirlds/merkledb/files/hashmap/BucketTest.java +++ b/platform-sdk/swirlds-jasperdb/src/test/java/com/swirlds/merkledb/files/hashmap/BucketTest.java @@ -86,7 +86,7 @@ void testBucketAddAndDelete(KeyType keyType) throws IOException { // insert keys and check for (int i = 0; i < 10; i++) { final VirtualLongKey key = testKeys[i]; - bucket.putValue(key.hashCode(), key, key.getKeyAsLong() + 100); + bucket.putValue(key, key.getKeyAsLong() + 100); assertEquals(i + 1, bucket.getBucketEntryCount(), "Check we have correct count"); // check that all keys added so far are there for (int j = 0; j <= i; j++) { @@ -95,15 +95,15 @@ void testBucketAddAndDelete(KeyType keyType) throws IOException { } assertEquals(10, bucket.getBucketEntryCount(), "Check we have correct count"); // now update, check and put back - bucket.putValue(testKeys[5].hashCode(), testKeys[5], 1234); + bucket.putValue(testKeys[5], 1234); assertEquals( 1234, bucket.findValue(testKeys[5].hashCode(), testKeys[5], -1), "Should get expected value of 1234"); - bucket.putValue(testKeys[5].hashCode(), testKeys[5], 115); + bucket.putValue(testKeys[5], 115); for (int j = 0; j < 10; j++) { checkKey(bucket, testKeys[j]); } // now delete last key and check - bucket.putValue(testKeys[9].hashCode(), testKeys[9], SPECIAL_DELETE_ME_VALUE); + bucket.putValue(testKeys[9], SPECIAL_DELETE_ME_VALUE); assertEquals(9, bucket.getBucketEntryCount(), "Check we have correct count"); for (int j = 0; j < 9; j++) { checkKey(bucket, testKeys[j]); @@ -113,7 +113,7 @@ void testBucketAddAndDelete(KeyType keyType) throws IOException { bucket.findValue(testKeys[9].hashCode(), testKeys[9], -1), "Should not find entry 10 any more we deleted it"); // now delete a middle, index 5 - bucket.putValue(testKeys[5].hashCode(), testKeys[5], SPECIAL_DELETE_ME_VALUE); + bucket.putValue(testKeys[5], SPECIAL_DELETE_ME_VALUE); assertEquals(8, bucket.getBucketEntryCount(), "Check we have correct count"); for (int j = 0; j < 5; j++) { checkKey(bucket, testKeys[j]); @@ -126,7 +126,7 @@ void testBucketAddAndDelete(KeyType keyType) throws IOException { bucket.findValue(testKeys[5].hashCode(), testKeys[5], -1), "Should not find entry 5 any more we deleted it"); // now delete first, index 0 - bucket.putValue(testKeys[0].hashCode(), testKeys[0], SPECIAL_DELETE_ME_VALUE); + bucket.putValue(testKeys[0], SPECIAL_DELETE_ME_VALUE); assertEquals(7, bucket.getBucketEntryCount(), "Check we have correct count"); for (int j = 1; j < 5; j++) { checkKey(bucket, testKeys[j]); @@ -139,8 +139,8 @@ void testBucketAddAndDelete(KeyType keyType) throws IOException { bucket.findValue(testKeys[0].hashCode(), testKeys[0], -1), "Should not find entry 0 any more we deleted it"); // add two more entries and check - bucket.putValue(testKeys[10].hashCode(), testKeys[10], 120); - bucket.putValue(testKeys[11].hashCode(), testKeys[11], 121); + bucket.putValue(testKeys[10], 120); + bucket.putValue(testKeys[11], 121); assertEquals(9, bucket.getBucketEntryCount(), "Check we have correct count"); for (int j = 1; j < 5; j++) { checkKey(bucket, testKeys[j]); @@ -152,9 +152,9 @@ void testBucketAddAndDelete(KeyType keyType) throws IOException { checkKey(bucket, testKeys[j]); } // put 0, 5 and 9 back in, and check we have full range - bucket.putValue(testKeys[0].hashCode(), testKeys[0], 110); - bucket.putValue(testKeys[5].hashCode(), testKeys[5], 115); - bucket.putValue(testKeys[9].hashCode(), testKeys[9], 119); + bucket.putValue(testKeys[0], 110); + bucket.putValue(testKeys[5], 115); + bucket.putValue(testKeys[9], 119); assertEquals(12, bucket.getBucketEntryCount(), "Check we have correct count"); for (int j = 0; j < 12; j++) { checkKey(bucket, testKeys[j]); @@ -177,7 +177,7 @@ void testBucketGrowing(KeyType keyType) { // insert keys and check for (int i = 0; i < testKeys.length; i++) { final VirtualLongKey key = testKeys[i]; - bucket.putValue(key.hashCode(), key, key.getKeyAsLong() + 100); + bucket.putValue(key, key.getKeyAsLong() + 100); assertEquals(i + 1, bucket.getBucketEntryCount(), "Check we have correct count"); // check that all keys added so far are there for (int j = 0; j <= i; j++) { @@ -203,7 +203,7 @@ void testBucketImportExportClear(KeyType keyType) throws IOException { // insert keys and check for (int i = 0; i < testKeys.length; i++) { final VirtualLongKey key = testKeys[i]; - bucket.putValue(key.hashCode(), key, key.getKeyAsLong() + 100); + bucket.putValue(key, key.getKeyAsLong() + 100); assertEquals(i + 1, bucket.getBucketEntryCount(), "Check we have correct count"); // check that all keys added so far are there for (int j = 0; j <= i; j++) { @@ -246,7 +246,7 @@ void toStringAsExpectedForBucket() { assertEquals(emptyBucketRepr, bucket.toString(), "Empty bucket should represent as expected"); final ExampleLongKeyFixedSize key = new ExampleLongKeyFixedSize(2056); - bucket.putValue(key.hashCode(), key, 5124); + bucket.putValue(key, 5124); bucket.setBucketIndex(0); final String nonEmptyBucketRepr = "Bucket{bucketIndex=0, entryCount=1, size=32\n" + " ENTRY[0] value= 5124 keyHashCode=2056 keyVer=3054" diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/AddressBookInitializer.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/AddressBookInitializer.java index 3e218ad8ed67..bb2bcda85197 100644 --- a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/AddressBookInitializer.java +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/AddressBookInitializer.java @@ -23,6 +23,7 @@ import com.swirlds.common.system.SwirldState; import com.swirlds.common.system.address.AddressBook; import com.swirlds.common.system.address.AddressBookValidator; +import com.swirlds.platform.config.AddressBookConfig; import com.swirlds.platform.state.signed.SignedState; import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; @@ -34,8 +35,10 @@ import java.time.Instant; import java.time.ZoneId; import java.time.format.DateTimeFormatter; +import java.util.List; import java.util.Objects; import java.util.function.Supplier; +import java.util.stream.Stream; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -56,8 +59,6 @@ public class AddressBookInitializer { public static final String STATE_ADDRESS_BOOK_USED = "The State Saved Address Book Was Used."; /** The text indicating the state address book was null in the usedAddressBook file. */ public static final String STATE_ADDRESS_BOOK_NULL = "The State Saved Address Book Was NULL."; - /** The name of the address book directory to write address books to. */ - private static final String ADDRESS_BOOK_DIRECTORY_NAME = "address_book"; /** The file name prefix to use when creating address book files. */ private static final String ADDRESS_BOOK_FILE_PREFIX = "usedAddressBook"; /** The format of date and time to use when creating address book files. */ @@ -86,6 +87,8 @@ public class AddressBookInitializer { /** The path to the directory for writing address books. */ @NonNull private final Path pathToAddressBookDirectory; + /** The maximum number of address book files to keep in the address book directory. */ + private final int maxNumFiles; /** Indicate that the unmodified config address book must be used. */ private final boolean useConfigAddressBook; @@ -97,31 +100,30 @@ public class AddressBookInitializer { * @param signedState The signed state loaded from disk. May be null. * @param genesisSupplier The swirld application state in genesis start. Must not be null. * @param configAddressBook The address book derived from config.txt. Must not be null. - * @param parentDirectory The parent directory of the address book directory. Must not be null. - * @param useConfigAddressBook Indicates if the unmodified config address book should be used. + * @param addressBookConfig The configuration settings for AddressBooks. */ public AddressBookInitializer( @NonNull final SoftwareVersion currentVersion, @Nullable final SignedState signedState, @NonNull final Supplier genesisSupplier, @NonNull final AddressBook configAddressBook, - @NonNull final Path parentDirectory, - final boolean useConfigAddressBook) { + @NonNull final AddressBookConfig addressBookConfig) { this.currentVersion = Objects.requireNonNull(currentVersion, "The currentVersion must not be null."); this.genesisSupplier = Objects.requireNonNull(genesisSupplier, "The genesis swirldState supplier must not be null."); this.configAddressBook = Objects.requireNonNull(configAddressBook, "The configAddressBook must not be null."); - Objects.requireNonNull(parentDirectory, "The parentDirectory must not be null."); - this.pathToAddressBookDirectory = parentDirectory.resolve(ADDRESS_BOOK_DIRECTORY_NAME); + Objects.requireNonNull(addressBookConfig, "The addressBookConfig must not be null."); + this.loadedSignedState = signedState; + this.loadedAddressBook = loadedSignedState == null ? null : loadedSignedState.getAddressBook(); + this.pathToAddressBookDirectory = Path.of(addressBookConfig.addressBookDirectory()); try { Files.createDirectories(pathToAddressBookDirectory); } catch (final IOException e) { logger.error(EXCEPTION.getMarker(), "Not able to create directory: {}", pathToAddressBookDirectory, e); throw new IllegalStateException("Not able to create directory: " + pathToAddressBookDirectory, e); } - this.loadedSignedState = signedState; - this.loadedAddressBook = loadedSignedState == null ? null : loadedSignedState.getAddressBook(); - this.useConfigAddressBook = useConfigAddressBook; + this.useConfigAddressBook = addressBookConfig.forceUseOfConfigAddressBook(); + this.maxNumFiles = addressBookConfig.maxRecordedAddressBookFiles(); initialAddressBook = initialize(); } @@ -151,6 +153,9 @@ public AddressBook getInitialAddressBook() { private AddressBook initialize() { AddressBook candidateAddressBook; if (useConfigAddressBook) { + logger.info( + STARTUP.getMarker(), + "Overriding the address book in the state with the address book from config.txt"); // configuration is overriding to force use of configuration address book. candidateAddressBook = configAddressBook; } else if (loadedSignedState == null || loadedAddressBook == null) { @@ -231,7 +236,7 @@ private AddressBook checkCandidateAddressBookValidity(@Nullable final AddressBoo * * @param usedAddressBook the address book to be returned from the AddressBookInitializer. */ - private void recordAddressBooks(@NonNull final AddressBook usedAddressBook) { + private synchronized void recordAddressBooks(@NonNull final AddressBook usedAddressBook) { final String date = DATE_TIME_FORMAT.format(Instant.now()); final String addressBookFileName = ADDRESS_BOOK_FILE_PREFIX + "_v" + currentVersion + "_" + date + ".txt"; final String addressBookDebugFileName = addressBookFileName + ".debug"; @@ -263,6 +268,23 @@ private void recordAddressBooks(@NonNull final AddressBook usedAddressBook) { } catch (final IOException e) { logger.error(EXCEPTION.getMarker(), "Not able to write address book to file. ", e); } + cleanAddressBookDirectory(); + } + + /** + * Deletes the oldest address book files if there are more than the maximum number of address book files. + */ + private synchronized void cleanAddressBookDirectory() { + try (final Stream filesStream = Files.list(pathToAddressBookDirectory)) { + final List files = filesStream.sorted().toList(); + if (files.size() > maxNumFiles) { + for (int i = 0; i < files.size() - maxNumFiles; i++) { + Files.delete(files.get(i)); + } + } + } catch (final IOException e) { + logger.info(EXCEPTION.getMarker(), "Unable to list files in address book directory. ", e); + } } /** diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/Browser.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/Browser.java index f57d620c9356..051922add747 100644 --- a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/Browser.java +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/Browser.java @@ -50,7 +50,6 @@ import com.swirlds.common.crypto.CryptographyHolder; import com.swirlds.common.crypto.config.CryptoConfig; import com.swirlds.common.internal.ApplicationDefinition; -import com.swirlds.common.internal.ConfigurationException; import com.swirlds.common.io.config.TemporaryFileConfig; import com.swirlds.common.merkle.synchronization.config.ReconnectConfig; import com.swirlds.common.metrics.Metrics; @@ -119,9 +118,12 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; import java.util.List; +import java.util.Map; +import java.util.Objects; import java.util.Set; import javax.swing.JFrame; import javax.swing.UIManager; @@ -182,7 +184,15 @@ private Browser(final Set localNodesToStart) throws IOException { final ConfigSource configPropertiesConfigSource = new ConfigPropertiesSource(configurationProperties); final ConfigSource configPropertiesAliasConfigSource = new AliasConfigSource(configPropertiesConfigSource); - configuration = ConfigurationBuilder.create() + // Load config.txt file, parse application jar file name, main class name, address book, and parameters + final ApplicationDefinition appDefinition = + ApplicationDefinitionLoader.load(configurationProperties, localNodesToStart); + + // Load all SwirldMain instances for locally run nodes. + final Map appMains = loadSwirldMains(appDefinition, localNodesToStart); + + // Load Configuration Definitions + final ConfigurationBuilder configurationBuilder = ConfigurationBuilder.create() .withSource(settingsAliasConfigSource) .withSource(configPropertiesAliasConfigSource) .withConfigDataType(BasicConfig.class) @@ -202,8 +212,17 @@ private Browser(final Set localNodesToStart) throws IOException { .withConfigDataType(PrometheusConfig.class) .withConfigDataType(OSHealthCheckConfig.class) .withConfigDataType(WiringConfig.class) - .withConfigDataType(PreConsensusEventStreamConfig.class) - .build(); + .withConfigDataType(PreConsensusEventStreamConfig.class); + + // Assume all locally run instances provide the same configuration definitions to the configuration builder. + if (appMains.size() > 0) { + appMains.values().iterator().next().updateConfigurationBuilder(configurationBuilder); + } + + this.configuration = configurationBuilder.build(); + + // Set the configuration on all SwirldMain instances. + appMains.values().forEach(swirldMain -> swirldMain.setConfiguration(configuration)); ConfigurationHolder.getInstance().setConfiguration(configuration); CryptographyHolder.reset(); @@ -273,7 +292,7 @@ private Browser(final Set localNodesToStart) throws IOException { } // instantiate all Platform objects, which each instantiates a Statistics object logger.debug(STARTUP.getMarker(), "About to run startPlatforms()"); - startPlatforms(configuration, configurationProperties, localNodesToStart); + startPlatforms(configuration, appDefinition, appMains); // create the browser window, which uses those Statistics objects showBrowserWindow(); @@ -324,6 +343,59 @@ private Browser(final Set localNodesToStart) throws IOException { logger.debug(STARTUP.getMarker(), "main() finished"); } + /** + * Load all {@link SwirldMain} instances for locally run nodes. Locally run nodes are indicated in two possible + * ways. One is through the set of local nodes to start. The other is through {@link Address::isOwnHost} being + * true. + * + * @param appDefinition the application definition + * @param localNodesToStart the locally run nodeIds + * @return a map from nodeIds to {@link SwirldMain} instances + * @throws AppLoaderException if there are issues loading the user app + * @throws ConstructableRegistryException if there are issues registering + * {@link com.swirlds.common.constructable.RuntimeConstructable} classes + */ + @NonNull + private Map loadSwirldMains( + @NonNull final ApplicationDefinition appDefinition, @NonNull final Set localNodesToStart) { + Objects.requireNonNull(appDefinition, "appDefinition must not be null"); + Objects.requireNonNull(localNodesToStart, "localNodesToStart must not be null"); + try { + // Create the SwirldAppLoader + final SwirldAppLoader appLoader; + try { + appLoader = + SwirldAppLoader.loadSwirldApp(appDefinition.getMainClassName(), appDefinition.getAppJarPath()); + } catch (final AppLoaderException e) { + CommonUtils.tellUserConsolePopup("ERROR", e.getMessage()); + throw e; + } + + // Register all RuntimeConstructable classes + logger.debug(STARTUP.getMarker(), "Scanning the classpath for RuntimeConstructable classes"); + final long start = System.currentTimeMillis(); + ConstructableRegistry.getInstance().registerConstructables("", appLoader.getClassLoader()); + logger.debug( + STARTUP.getMarker(), + "Done with registerConstructables, time taken {}ms", + System.currentTimeMillis() - start); + + // Create the SwirldMain instances + final Map appMains = new HashMap<>(); + final AddressBook addressBook = appDefinition.getAddressBook(); + for (int i = 0; i < addressBook.getSize(); i++) { + final long id = addressBook.getId(i); + final Address address = addressBook.getAddress(id); + if (localNodesToStart.contains((int) id) || address.isOwnHost()) { + appMains.put(id, buildAppMain(appDefinition, appLoader)); + } + } + return appMains; + } catch (final Exception ex) { + throw new RuntimeException("Error loading SwirldMains", ex); + } + } + /** * Writes all settings and config values to settingsUsed.txt * @@ -517,9 +589,15 @@ private List createLocalPlatforms( @NonNull final ApplicationDefinition appDefinition, @NonNull final Crypto[] crypto, @NonNull final InfoSwirld infoSwirld, - @NonNull final SwirldAppLoader appLoader, + @NonNull final Map appMains, @NonNull final Configuration configuration, @NonNull final MetricsProvider metricsProvider) { + Objects.requireNonNull(appDefinition, "the app definition must not be null"); + Objects.requireNonNull(crypto, "the crypto array must not be null"); + Objects.requireNonNull(infoSwirld, "the infoSwirld must not be null"); + Objects.requireNonNull(appMains, "the appMains map must not be null"); + Objects.requireNonNull(configuration, "the configuration must not be null"); + Objects.requireNonNull(metricsProvider, "the metricsProvider must not be null"); final List platforms = new ArrayList<>(); @@ -541,8 +619,7 @@ private List createLocalPlatforms( final PlatformContext platformContext = new DefaultPlatformContext(nodeId, metricsProvider, configuration); - final SwirldMain appMain = buildAppMain(appDefinition, appLoader); - appMain.setConfiguration(configuration); + SwirldMain appMain = appMains.get(address.getId()); // name of the app's SwirldMain class final String mainClassName = appDefinition.getMainClassName(); @@ -557,17 +634,12 @@ private List createLocalPlatforms( final SignedState loadedSignedState = getUnmodifiedSignedStateFromDisk( mainClassName, swirldName, nodeId, appVersion, addressBook.copy(), emergencyRecoveryManager); - final StateConfig stateConfig = - platformContext.getConfiguration().getConfigData(StateConfig.class); + final AddressBookConfig addressBookConfig = + platformContext.getConfiguration().getConfigData(AddressBookConfig.class); // Initialize the address book from the configuration and platform saved state. final AddressBookInitializer addressBookInitializer = new AddressBookInitializer( - appVersion, - loadedSignedState, - appMain::newState, - addressBook.copy(), - stateConfig.savedStateDirectory(), - stateConfig.forceUseOfConfigAddressBook()); + appVersion, loadedSignedState, appMain::newState, addressBook.copy(), addressBookConfig); // set here, then given to the state in run(). A copy of it is given to hashgraph. final AddressBook initialAddressBook = addressBookInitializer.getInitialAddressBook(); @@ -619,13 +691,13 @@ private List createLocalPlatforms( /** * Load the signed state from the disk if it is present. * - * @param mainClassName the name of the app's SwirldMain class. - * @param swirldName the name of the swirld to load the state for. - * @param selfId the ID of the node to load the state for. - * @param appVersion the version of the app to use for emergency recovery. - * @param configAddressBook the address book to use for emergency recovery. + * @param mainClassName the name of the app's SwirldMain class. + * @param swirldName the name of the swirld to load the state for. + * @param selfId the ID of the node to load the state for. + * @param appVersion the version of the app to use for emergency recovery. + * @param configAddressBook the address book to use for emergency recovery. * @param emergencyRecoveryManager the emergency recovery manager to use for emergency recovery. - * @return the signed state loaded from disk. + * @return the signed state loaded from disk. */ private SignedState getUnmodifiedSignedStateFromDisk( @NonNull final String mainClassName, @@ -659,27 +731,15 @@ private SignedState getUnmodifiedSignedStateFromDisk( * Instantiate and run all the local platforms specified in the given config.txt file. This method reads in and * parses the config.txt file. * - * @throws UnknownHostException problems getting an IP address for another user - * @throws SocketException problems getting the IP address for self - * @throws AppLoaderException if there are issues loading the user app - * @throws ConstructableRegistryException if there are issues registering - * {@link com.swirlds.common.constructable.RuntimeConstructable} classes + * @throws UnknownHostException problems getting an IP address for another user + * @throws SocketException problems getting the IP address for self */ private void startPlatforms( - final Configuration configuration, - final LegacyConfigProperties configurationProperties, - final Set localNodesToStart) - throws AppLoaderException, ConstructableRegistryException { + @NonNull final Configuration configuration, + @NonNull final ApplicationDefinition appDefinition, + @NonNull final Map appMains) { - // Load config.txt file, parse application jar file name, main class name, address book, and parameters - final ApplicationDefinition appDefinition; - final AddressBook addressBook; - try { - appDefinition = ApplicationDefinitionLoader.load(configurationProperties, localNodesToStart); - addressBook = appDefinition.getAddressBook(); - } catch (final ConfigurationException ex) { - return; - } + final AddressBook addressBook = appDefinition.getAddressBook(); // If enabled, clean out the signed state directory. Needs to be done before the platform/state is started up, // as we don't want to delete the temporary file directory if it ends up being put in the saved state directory. @@ -726,24 +786,6 @@ private void startPlatforms( logger.debug(STARTUP.getMarker(), "Starting platforms"); - // Try to load the app - final SwirldAppLoader appLoader; - try { - appLoader = SwirldAppLoader.loadSwirldApp(appDefinition.getMainClassName(), appDefinition.getAppJarPath()); - } catch (final AppLoaderException e) { - CommonUtils.tellUserConsolePopup("ERROR", e.getMessage()); - throw e; - } - - // Register all RuntimeConstructable classes - logger.debug(STARTUP.getMarker(), "Scanning the classpath for RuntimeConstructable classes"); - final long start = System.currentTimeMillis(); - ConstructableRegistry.getInstance().registerConstructables("", appLoader.getClassLoader()); - logger.debug( - STARTUP.getMarker(), - "Done with registerConstructables, time taken {}ms", - System.currentTimeMillis() - start); - // Setup metrics system final DefaultMetricsProvider metricsProvider = new DefaultMetricsProvider(configuration); final Metrics globalMetrics = metricsProvider.createGlobalMetrics(); @@ -751,7 +793,7 @@ private void startPlatforms( // Create all instances for all nodes that should run locally final List platforms = - createLocalPlatforms(appDefinition, crypto, infoSwirld, appLoader, configuration, metricsProvider); + createLocalPlatforms(appDefinition, crypto, infoSwirld, appMains, configuration, metricsProvider); // Write all metrics information to file MetricsDocUtils.writeMetricsDocumentToFile(globalMetrics, getPlatforms(), configuration); diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/cli/EventStreamSignCommand.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/cli/EventStreamSignCommand.java new file mode 100644 index 000000000000..aa9819b3a5b7 --- /dev/null +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/cli/EventStreamSignCommand.java @@ -0,0 +1,61 @@ +/* + * Copyright (C) 2023 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.swirlds.platform.cli; + +import com.swirlds.cli.commands.EventStreamCommand; +import com.swirlds.cli.utility.SubcommandOf; +import com.swirlds.common.stream.EventStreamType; +import com.swirlds.platform.util.EventStreamSigningUtils; +import edu.umd.cs.findbugs.annotations.NonNull; +import java.nio.file.Path; +import java.security.KeyPair; +import picocli.CommandLine; + +/** + * A subcommand of the {@link SignCommand}, for signing event stream files + */ +@CommandLine.Command(name = "sign", mixinStandardHelpOptions = true, description = "Sign event stream files") +@SubcommandOf(EventStreamCommand.class) +public final class EventStreamSignCommand extends SignCommand { + /** + * {@inheritDoc} + */ + @Override + public boolean generateSignatureFile( + @NonNull Path signatureFileDestination, @NonNull Path fileToSign, @NonNull KeyPair keyPair) { + + return EventStreamSigningUtils.signEventStreamFile(signatureFileDestination, fileToSign, keyPair); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean isFileSupported(@NonNull final Path path) { + return EventStreamType.getInstance().isStreamFile(path.toFile()); + } + + /** + * {@inheritDoc} + */ + @Override + public Integer call() { + EventStreamSigningUtils.initializeSystem(); + + return super.call(); + } +} diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/cli/SignCommand.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/cli/SignCommand.java new file mode 100644 index 000000000000..91553bb0715e --- /dev/null +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/cli/SignCommand.java @@ -0,0 +1,196 @@ +/* + * Copyright (C) 2023 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.swirlds.platform.cli; + +import com.swirlds.cli.utility.AbstractCommand; +import com.swirlds.common.io.utility.FileUtils; +import com.swirlds.platform.util.FileSigningUtils; +import edu.umd.cs.findbugs.annotations.NonNull; +import edu.umd.cs.findbugs.annotations.Nullable; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.KeyPair; +import java.util.List; +import java.util.stream.Stream; +import picocli.CommandLine; + +/** + * An abstract command type for generating signature files + */ +public abstract class SignCommand extends AbstractCommand { + /** + * The paths to the sources that signature files will be generated for. Can contain individual files, as well as + * directories + *

+ * If an element of this list is a directory, then all files in that directory will be signed, recursively + *

+ * Defaults to the current working directory + */ + private List pathsToSign = List.of(FileUtils.getAbsolutePath()); + + /** + * The directory where signature files will be generated. Defaults to in-place signatures (null) + */ + @Nullable + private Path destinationDirectory = null; + + /** + * The path to the key file to use to generate signatures + */ + private Path keyFilePath; + + /** + * The password to the key file + */ + private String keyFilePassword; + + /** + * The alias of the key in the key file + */ + private String keyAlias; + + /** + * The key pair to use to generate signatures + */ + private KeyPair keyPair; + + @CommandLine.Parameters(description = "The path to the key file to use to generate signatures", index = "0") + private void setKeyFile(@NonNull final Path keyFilePath) { + this.keyFilePath = pathMustExist(keyFilePath.toAbsolutePath().normalize()); + } + + @CommandLine.Parameters(description = "The password to the key file", index = "1") + private void setKeyFilePassword(@NonNull final String keyFilePassword) { + this.keyFilePassword = keyFilePassword; + } + + @CommandLine.Parameters(description = "The alias of the key in the key file", index = "2") + private void setKeyAlias(@NonNull final String keyAlias) { + this.keyAlias = keyAlias; + } + + @CommandLine.Option( + names = {"-p", "--paths-to-sign"}, + description = "The paths to what will be signed. Can contain single files, as well as directories." + + "Defaults to the current working directory") + private void setPathsToSign(@NonNull final List pathsToSign) { + this.pathsToSign = pathsToSign.stream() + .map((path -> pathMustExist(path.toAbsolutePath().normalize()))) + .toList(); + } + + @CommandLine.Option( + names = {"-d", "--destination-directory"}, + description = "Specify the destination directory where signature files will be generated. If specified," + + "source files will be copied to the destination directory. If not specified," + + "the signature file will simply be generated in the same directory as the source file") + private void setDestinationDirectory(@NonNull final Path destinationDirectory) { + this.destinationDirectory = destinationDirectory.toAbsolutePath().normalize(); + } + + /** + * {@inheritDoc} + */ + @Override + public Integer call() { + keyPair = FileSigningUtils.loadPfxKey(keyFilePath, keyFilePassword, keyAlias); + + pathsToSign.forEach(this::signFilesAtPath); + + return 0; + } + + /** + * Generate a signature file for a single source file + * + * @param signatureFileDestination the full path where the signature file will be generated + * @param fileToSign the file to generate a signature file for + * @param keyPair the key pair to use to generate the signature + * @return true if the signature file was generated successfully, false otherwise + */ + public abstract boolean generateSignatureFile( + @NonNull final Path signatureFileDestination, + @NonNull final Path fileToSign, + @NonNull final KeyPair keyPair); + + /** + * Check if a file is supported by this command + * + * @param path the path to the file to check + * @return true if the file is supported, false otherwise + */ + public abstract boolean isFileSupported(@NonNull final Path path); + + /** + * Perform necessary tasks to sign a single file. Generates a signature file via {@link #generateSignatureFile} if + * one doesn't already exist + *

+ * If a destinationDirectory has been specified, the source file will additionally be copied to the destination + * directory + * + * @param fileToSign the file to generate a signature file for + */ + private void signFile(@NonNull final Path fileToSign) { + final Path signatureFileDestinationPath; + if (destinationDirectory == null) { + // if destinationDirectory is null, then we are generating in-place signatures + signatureFileDestinationPath = FileSigningUtils.buildSignatureFilePath(fileToSign.getParent(), fileToSign); + } else { + signatureFileDestinationPath = FileSigningUtils.buildSignatureFilePath(destinationDirectory, fileToSign); + } + + // if signature file already exists, don't continue + if (Files.exists(signatureFileDestinationPath)) { + System.out.println( + "Signature file " + signatureFileDestinationPath + " already exists. Skipping file " + fileToSign); + return; + } + + // if signature generation fails, don't continue + if (!generateSignatureFile(signatureFileDestinationPath, fileToSign, keyPair)) { + // don't print anything here, as a specific error message should be printed by the signing implementation + return; + } + + // if input destinationDirectory was null, then we generated in-place signatures. No need to copy source files + if (destinationDirectory == null) { + return; + } + + try { + Files.copy(fileToSign, destinationDirectory.resolve(fileToSign.getFileName())); + } catch (final IOException e) { + System.err.println("Failed to copy source file " + fileToSign.getFileName() + " to destination directory " + + destinationDirectory + ". Exception: " + e); + } + } + + /** + * Sign file(s) at a path. Individual files will simply be signed, and directories will have all supported files + * signed, recursively + * + * @param path the path to sign files at + */ + private void signFilesAtPath(@NonNull final Path path) { + try (final Stream stream = Files.walk(path)) { + stream.filter(this::isFileSupported).forEach(this::signFile); + } catch (final IOException e) { + throw new RuntimeException("Failed to list files: " + path); + } + } +} diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/config/AddressBookConfig.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/config/AddressBookConfig.java index 2adbd94bcf97..6800f9e39980 100644 --- a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/config/AddressBookConfig.java +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/config/AddressBookConfig.java @@ -27,6 +27,20 @@ * Intermediate workaround until the platform is capable of handling an address book that changes at runtime. This * feature will be removed in a future version. Check if the address book override is enabled. If enabled, then * the address book will only change when a node starts up. This feature will be removed in a future version. + * @param forceUseOfConfigAddressBook + * If true, then the address book from the config file will be used instead of the address book from the + * signed state and the swirld state will not be queried for any address book updates. + * @param addressBookDirectory + * The directory where address book files are saved. + * @param maxRecordedAddressBookFiles + * The maximum number of address book files to keep in the address book directory. On startup the + * AddressBookInitializer will create two new files in the `/data/saved/address_book` directory. The + * AddressBookInitializer will delete files by age, oldest first, if the number of files in the directory + * exceeds the maximum indicated by this setting. */ @ConfigData("addressBook") -public record AddressBookConfig(@ConfigProperty(defaultValue = "true") boolean updateAddressBookOnlyAtUpgrade) {} +public record AddressBookConfig( + @ConfigProperty(defaultValue = "true") boolean updateAddressBookOnlyAtUpgrade, + @ConfigProperty(defaultValue = "true") boolean forceUseOfConfigAddressBook, + @ConfigProperty(defaultValue = "data/saved/address_book") String addressBookDirectory, + @ConfigProperty(defaultValue = "50") int maxRecordedAddressBookFiles) {} diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/sync/SyncFallenBehindStatus.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/sync/SyncFallenBehindStatus.java index b89b9afb756d..021957500666 100644 --- a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/sync/SyncFallenBehindStatus.java +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/sync/SyncFallenBehindStatus.java @@ -24,10 +24,10 @@ public enum SyncFallenBehindStatus { OTHER_FALLEN_BEHIND; public static SyncFallenBehindStatus getStatus(final GraphGenerations self, final GraphGenerations other) { - if (other.getMaxRoundGeneration() < self.getMinRoundGeneration()) { + if (other.getMaxRoundGeneration() < self.getMinGenerationNonAncient()) { return OTHER_FALLEN_BEHIND; } - if (self.getMaxRoundGeneration() < other.getMinRoundGeneration()) { + if (self.getMaxRoundGeneration() < other.getMinGenerationNonAncient()) { return SELF_FALLEN_BEHIND; } return NONE_FALLEN_BEHIND; diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/util/EventStreamSigningUtils.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/util/EventStreamSigningUtils.java index 3948fba3a127..92d5a662beb0 100644 --- a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/util/EventStreamSigningUtils.java +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/util/EventStreamSigningUtils.java @@ -20,7 +20,6 @@ import static com.swirlds.common.stream.LinkedObjectStreamUtilities.computeMetaHash; import static com.swirlds.common.stream.LinkedObjectStreamUtilities.readFirstIntFromFile; import static com.swirlds.common.stream.internal.TimestampStreamFileWriter.writeSignatureFile; -import static com.swirlds.platform.util.FileSigningUtils.buildSignatureFilePath; import static com.swirlds.platform.util.FileSigningUtils.signData; import com.swirlds.common.crypto.Hash; @@ -30,7 +29,6 @@ import com.swirlds.common.stream.internal.InvalidStreamFileException; import edu.umd.cs.findbugs.annotations.NonNull; import java.io.IOException; -import java.nio.file.Files; import java.nio.file.Path; import java.security.InvalidKeyException; import java.security.KeyPair; @@ -38,7 +36,6 @@ import java.security.NoSuchProviderException; import java.security.SignatureException; import java.util.Objects; -import java.util.stream.Stream; /** * Utility class for signing event stream files @@ -74,16 +71,17 @@ public static void initializeSystem() { /** * Generates a signature file for the given event stream file * - * @param destinationDirectory the directory where the signature file will be saved - * @param streamFileToSign the stream file to be signed - * @param keyPair the keyPair used for signing + * @param signatureFileDestination the full path where the signature file will be generated + * @param streamFileToSign the stream file to be signed + * @param keyPair the keyPair used for signing + * @return true if the signature file was generated successfully, false otherwise */ - public static void signEventStreamFile( - @NonNull final Path destinationDirectory, + public static boolean signEventStreamFile( + @NonNull final Path signatureFileDestination, @NonNull final Path streamFileToSign, @NonNull final KeyPair keyPair) { - Objects.requireNonNull(destinationDirectory, "destinationDirectory must not be null"); + Objects.requireNonNull(signatureFileDestination, "signatureFileDestination must not be null"); Objects.requireNonNull(streamFileToSign, "streamFileToSign must not be null"); Objects.requireNonNull(keyPair, "keyPair must not be null"); @@ -93,7 +91,7 @@ public static void signEventStreamFile( System.err.printf( "Failed to sign file [%s] with unsupported version [%s]%n", streamFileToSign.getFileName(), version); - return; + return false; } final Hash entireHash = computeEntireHash(streamFileToSign.toFile()); @@ -106,50 +104,22 @@ public static void signEventStreamFile( final com.swirlds.common.crypto.Signature metaHashSignature = new com.swirlds.common.crypto.Signature(SignatureType.RSA, signData(metaHash.getValue(), keyPair)); - final Path signatureFilePath = buildSignatureFilePath(destinationDirectory, streamFileToSign); - writeSignatureFile( entireHash, entireHashSignature, metaHash, metaHashSignature, - signatureFilePath.toString(), + signatureFileDestination.toString(), streamType); - System.out.println("Generated signature file: " + signatureFilePath); + System.out.println("Generated signature file: " + signatureFileDestination); + + return true; } catch (final SignatureException | InvalidStreamFileException | IOException e) { System.err.println("Failed to sign file " + streamFileToSign.getFileName() + ". Exception: " + e); + return false; } catch (final InvalidKeyException | NoSuchAlgorithmException | NoSuchProviderException e) { throw new RuntimeException("Irrecoverable error encountered", e); } } - - /** - * Signs all event stream files in a directory - *

- * If a recoverable error is encountered while signing an individual file in the directory, an error will be logged, - * and signing of remaining files will continue - * - * @param sourceDirectory the source directory - * @param destinationDirectory the destination directory - * @param keyPair the key pair to sign with - */ - public static void signEventStreamFilesInDirectory( - @NonNull final Path sourceDirectory, - @NonNull final Path destinationDirectory, - @NonNull final KeyPair keyPair) { - - Objects.requireNonNull(sourceDirectory, "sourceDirectory must not be null"); - Objects.requireNonNull(destinationDirectory, "destinationDirectory must not be null"); - Objects.requireNonNull(keyPair, "keyPair must not be null"); - - final EventStreamType streamType = EventStreamType.getInstance(); - - try (final Stream stream = Files.walk(sourceDirectory)) { - stream.filter(filePath -> streamType.isStreamFile(filePath.toString())) - .forEach(path -> signEventStreamFile(destinationDirectory, path, keyPair)); - } catch (final IOException e) { - throw new RuntimeException("Failed to list files in directory: " + sourceDirectory); - } - } } diff --git a/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/AddressBookInitializerTest.java b/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/AddressBookInitializerTest.java index f1b38cfaf586..94d008160182 100644 --- a/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/AddressBookInitializerTest.java +++ b/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/AddressBookInitializerTest.java @@ -36,10 +36,13 @@ import com.swirlds.common.system.address.Address; import com.swirlds.common.system.address.AddressBook; import com.swirlds.common.test.RandomAddressBookGenerator; +import com.swirlds.config.api.Configuration; +import com.swirlds.platform.config.AddressBookConfig; import com.swirlds.platform.state.PlatformData; import com.swirlds.platform.state.PlatformState; import com.swirlds.platform.state.State; import com.swirlds.platform.state.signed.SignedState; +import com.swirlds.test.framework.config.TestConfigBuilder; import java.io.File; import java.io.IOException; import java.nio.file.Files; @@ -70,8 +73,7 @@ void forceUseOfConfigAddressBookNullSignedState() throws IOException { getMockSignedState(0), getMockSwirldStateSupplier(1), configAddressBook, - testDirectory, - true); + getAddressBookConfig(true)); final AddressBook inititializedAddressBook = initializer.getInitialAddressBook(); assertEquals( configAddressBook, @@ -91,8 +93,7 @@ void forceUseOfConfigAddressBookNonNullSignedState() throws IOException { signedState, getMockSwirldStateSupplier(1), configAddressBook, - testDirectory, - true); + getAddressBookConfig(true)); final AddressBook inititializedAddressBook = initializer.getInitialAddressBook(); assertEquals( configAddressBook, @@ -113,8 +114,7 @@ void noStateLoadedFromDisk() throws IOException { getMockSignedState(0), getMockSwirldStateSupplier(10), configAddressBook, - testDirectory, - false); + getAddressBookConfig(false)); final AddressBook inititializedAddressBook = initializer.getInitialAddressBook(); assertEquals( expectedAddressBook, @@ -133,8 +133,7 @@ void noStateLoadedFromDiskGenesisStateSetZeroStake() throws IOException { getMockSignedState(0), getMockSwirldStateSupplier(0), configAddressBook, - testDirectory, - false); + getAddressBookConfig(false)); final AddressBook inititializedAddressBook = initializer.getInitialAddressBook(); assertEquals( configAddressBook, @@ -153,8 +152,7 @@ void noStateLoadedFromDiskGenesisStateChangedAddressBook() throws IOException { getMockSignedState(0), getMockSwirldStateSupplier(2), configAddressBook, - testDirectory, - false); + getAddressBookConfig(false)); final AddressBook inititializedAddressBook = initializer.getInitialAddressBook(); assertEquals( configAddressBook, @@ -171,16 +169,12 @@ void currentVersionLessThanStateVersion() throws IOException { final AddressBook configAddressBook = getRandomAddressBook(); final SoftwareVersion configSoftwareVersion = getMockSoftwareVersion(1); final Supplier genesisSwirldState = getMockSwirldStateSupplier(1); + final AddressBookConfig addressBookConfig = getAddressBookConfig(false); assertThrows( IllegalStateException.class, () -> new AddressBookInitializer( - configSoftwareVersion, - signedState, - genesisSwirldState, - configAddressBook, - testDirectory, - false), + configSoftwareVersion, signedState, genesisSwirldState, configAddressBook, addressBookConfig), "IllegalStateException was not thrown."); } @@ -195,8 +189,7 @@ void currentVersionEqualsStateVersion() throws IOException { signedState, getMockSwirldStateSupplier(1), configAddressBook, - testDirectory, - false); + getAddressBookConfig(false)); final AddressBook inititializedAddressBook = initializer.getInitialAddressBook(); assertEquals( signedState.getAddressBook(), @@ -217,8 +210,7 @@ void versionUpgradeSwirldStateZeroStake() throws IOException { signedState, getMockSwirldStateSupplier(0), configAddressBook, - testDirectory, - false); + getAddressBookConfig(false)); final AddressBook inititializedAddressBook = initializer.getInitialAddressBook(); assertEquals( configAddressBook, @@ -239,8 +231,7 @@ void versionUpgradeSwirldStateModifiedAddressBook() throws IOException { signedState, getMockSwirldStateSupplier(2), configAddressBook, - testDirectory, - false); + getAddressBookConfig(false)); final AddressBook inititializedAddressBook = initializer.getInitialAddressBook(); assertEquals( configAddressBook, @@ -261,8 +252,7 @@ void versionUpgradeSwirldStateStakeUpdateWorks() throws IOException { signedState, getMockSwirldStateSupplier(10), configAddressBook, - testDirectory, - false); + getAddressBookConfig(false)); final AddressBook inititializedAddressBook = initializer.getInitialAddressBook(); assertNotEquals( configAddressBook, @@ -500,4 +490,20 @@ private void assertAddressBookFileContent( debugFileContent.contains(usedText), "The usedAddressBook content is not:\n" + usedText + "\n\n debugFileContent:\n" + debugFileContent); } + + /** + * Creates an AddressBookConfig object with the given forceUseOfConfigAddressBook value. + * + * @param forceUseOfConfigAddressBook the value to use for the forceUseOfConfigAddressBook property. + * @return the AddressBookConfig object. + */ + private AddressBookConfig getAddressBookConfig(boolean forceUseOfConfigAddressBook) { + final Configuration configuration = new TestConfigBuilder() + .withValue("addressBook.forceUseOfConfigAddressBook", forceUseOfConfigAddressBook) + .withValue("addressBook.addressBookDirectory", testDirectory.toString()) + .withValue("addressBook.maxRecordedAddressBookFiles", 50) + .getOrCreateConfig(); + final AddressBookConfig addressBookConfig = configuration.getConfigData(AddressBookConfig.class); + return addressBookConfig; + } } diff --git a/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/util/EventStreamSigningUtilsTests.java b/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/util/EventStreamSigningUtilsTests.java index 0e9ffb197e04..078dd6a00d7f 100644 --- a/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/util/EventStreamSigningUtilsTests.java +++ b/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/util/EventStreamSigningUtilsTests.java @@ -22,7 +22,6 @@ import static com.swirlds.platform.recovery.RecoveryTestUtils.writeRandomEventStream; import static com.swirlds.platform.util.EventStreamSigningUtils.initializeSystem; import static com.swirlds.platform.util.EventStreamSigningUtils.signEventStreamFile; -import static com.swirlds.platform.util.EventStreamSigningUtils.signEventStreamFilesInDirectory; import static com.swirlds.platform.util.FileSigningUtils.SIGNATURE_FILE_NAME_SUFFIX; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -144,7 +143,10 @@ void signSingleStreamFile() { throw new RuntimeException(e); } - signEventStreamFile(destinationDirectory, fileToSign, keyPair); + assertTrue( + signEventStreamFile( + FileSigningUtils.buildSignatureFilePath(destinationDirectory, fileToSign), fileToSign, keyPair), + "Signing failed"); final List destinationDirectoryFiles = getDestinationDirectoryFiles(); @@ -162,42 +164,4 @@ void signSingleStreamFile() { validateFileAndSignature( fileToSign.toFile(), signatureFile.toFile(), keyPair.getPublic(), EventStreamType.getInstance()); } - - @Test - @DisplayName("Sign stream files in directory") - void signStreamFiles() { - createStreamFiles(); - - final List filesToSign; - try (final Stream stream = Files.walk(toSignDirectory)) { - filesToSign = stream.filter( - filePath -> EventStreamType.getInstance().isStreamFile(filePath.toString())) - .toList(); - } catch (final IOException e) { - throw new RuntimeException("Failed to list files in directory: " + toSignDirectory, e); - } - - signEventStreamFilesInDirectory(toSignDirectory, destinationDirectory, keyPair); - - final List destinationDirectoryFiles = getDestinationDirectoryFiles(); - - assertNotNull(destinationDirectoryFiles, "Expected signature file to be created"); - assertEquals( - filesToSign.size(), - destinationDirectoryFiles.size(), - "Expected correct number of signature files to be created"); - - for (final Path originalFile : filesToSign) { - final Path expectedFile = - destinationDirectory.resolve(originalFile.getFileName() + SIGNATURE_FILE_NAME_SUFFIX); - - assertTrue( - destinationDirectoryFiles.contains(expectedFile), - "Expected signature file to be created for " + originalFile.getFileName()); - - // validate the stream file and sig file via the standard method - validateFileAndSignature( - originalFile.toFile(), expectedFile.toFile(), keyPair.getPublic(), EventStreamType.getInstance()); - } - } } diff --git a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/sync/SyncTestExecutor.java b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/sync/SyncTestExecutor.java index 56163b32b6bd..3b8bb51f3af3 100644 --- a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/sync/SyncTestExecutor.java +++ b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/sync/SyncTestExecutor.java @@ -180,29 +180,13 @@ private BiConsumer getDefaultGenerationDefinitions() { long callerMinGen = SyncUtils.getMinGen(caller.getShadowGraph().findAncestors(callerTips, (e) -> true)); long callerMaxGen = SyncUtils.getMaxGen(callerTips); - long listenerMinNonAncientGen = listenerMinGen; - double listenerDif = listenerMaxGen - listenerMinGen; - if (listenerDif >= 3) { - listenerMinNonAncientGen += Math.floor(listenerDif / 3); - } else if (listenerDif == 2) { - listenerMinNonAncientGen++; - } - - long callerMinNonAncientGen = callerMinGen; - double callerDif = callerMaxGen - callerMinGen; - if (callerDif >= 3) { - callerMinNonAncientGen += Math.floor(callerDif / 3); - } else if (callerDif == 2) { - callerMinNonAncientGen++; - } - when(caller.getConsensus().getMaxRoundGeneration()).thenReturn(callerMaxGen); when(caller.getConsensus().getMinRoundGeneration()).thenReturn(callerMinGen); - when(caller.getConsensus().getMinGenerationNonAncient()).thenReturn(callerMinNonAncientGen); + when(caller.getConsensus().getMinGenerationNonAncient()).thenReturn(callerMinGen); when(listener.getConsensus().getMaxRoundGeneration()).thenReturn(listenerMaxGen); when(listener.getConsensus().getMinRoundGeneration()).thenReturn(listenerMinGen); - when(listener.getConsensus().getMinGenerationNonAncient()).thenReturn(listenerMinNonAncientGen); + when(listener.getConsensus().getMinGenerationNonAncient()).thenReturn(listenerMinGen); }; } diff --git a/settings.gradle.kts b/settings.gradle.kts index 0dba39663a9c..5ea440ad0ea7 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -99,7 +99,7 @@ gitRepositories { uri.set("https://github.com/hashgraph/hedera-protobufs.git") // choose tag or branch of HAPI you would like to test with // This version needs to match tha HAPI version below in versionCatalogs - tag.set("05111-use-pbj-for-hcs-state") + tag.set("v0.36.1") // do not load project from repo autoInclude.set(false) } @@ -114,7 +114,7 @@ dependencyResolutionManagement { // runtime. create("libs") { // The HAPI API version to use, this need to match the tag set on gitRepositories above - version("hapi-version", "0.36.0-alpha.1-SNAPSHOT") + version("hapi-version", "0.36.1") // Definition of version numbers for all libraries version("pbj-version", "0.3.0")