etcm-173 support for checkpoints in difficulty calculator

src/ets/scala/io/iohk/ethereum/ets/blockchain/BlockchainTestConfig.scala

@@ -3,7 +3,7 @@ package io.iohk.ethereum.ets.blockchain
 import akka.util.ByteString
 import io.iohk.ethereum.consensus.Protocol
 import io.iohk.ethereum.consensus.ethash.validators.ValidatorsExecutor
-import io.iohk.ethereum.domain.{Address, UInt256}
+import io.iohk.ethereum.domain.{Address, Blockchain, UInt256}
 import io.iohk.ethereum.utils.{BlockchainConfig, DaoForkConfig, MonetaryPolicyConfig}
 import org.bouncycastle.util.encoders.Hex
 
@@ -291,40 +291,40 @@ object BlockchainTestConfig {
   )
 }
 
-object Validators {
+class Validators(blockchain: Blockchain) {
   import BlockchainTestConfig._
 
-  val frontierValidators = ValidatorsExecutor(FrontierConfig, Protocol.Ethash)
-  val homesteadValidators = ValidatorsExecutor(HomesteadConfig, Protocol.Ethash)
-  val eip150Validators = ValidatorsExecutor(Eip150Config, Protocol.Ethash)
-  val frontierToHomesteadValidators = ValidatorsExecutor(FrontierToHomesteadAt5, Protocol.Ethash)
-  val homesteadToEipValidators = ValidatorsExecutor(HomesteadToEIP150At5, Protocol.Ethash)
-  val homesteadToDaoValidators= ValidatorsExecutor(HomesteadToDaoAt5, Protocol.Ethash)
-  val eip158Validators = ValidatorsExecutor(Eip158Config, Protocol.Ethash)
-  val byzantiumValidators = ValidatorsExecutor(ByzantiumConfig, Protocol.Ethash)
-  val constantinopleValidators = ValidatorsExecutor(ConstantinopleConfig, Protocol.Ethash)
-  val constantinopleFixValidators = ValidatorsExecutor(ConstantinopleFixConfig, Protocol.Ethash)
-  val istanbulValidators = ValidatorsExecutor(IstanbulConfig, Protocol.Ethash)
-  val eip158ToByzantiumValidators = ValidatorsExecutor(Eip158ToByzantiumAt5Config, Protocol.Ethash)
-  val byzantiumToConstantinopleAt5 = ValidatorsExecutor(ByzantiumToConstantinopleAt5, Protocol.Ethash)
+  val frontierValidators = ValidatorsExecutor(FrontierConfig, blockchain, Protocol.Ethash)
+  val homesteadValidators = ValidatorsExecutor(HomesteadConfig, blockchain, Protocol.Ethash)
+  val eip150Validators = ValidatorsExecutor(Eip150Config, blockchain, Protocol.Ethash)
+  val frontierToHomesteadValidators = ValidatorsExecutor(FrontierToHomesteadAt5, blockchain, Protocol.Ethash)
+  val homesteadToEipValidators = ValidatorsExecutor(HomesteadToEIP150At5, blockchain, Protocol.Ethash)
+  val homesteadToDaoValidators= ValidatorsExecutor(HomesteadToDaoAt5, blockchain, Protocol.Ethash)
+  val eip158Validators = ValidatorsExecutor(Eip158Config, blockchain, Protocol.Ethash)
+  val byzantiumValidators = ValidatorsExecutor(ByzantiumConfig, blockchain, Protocol.Ethash)
+  val constantinopleValidators = ValidatorsExecutor(ConstantinopleConfig, blockchain, Protocol.Ethash)
+  val constantinopleFixValidators = ValidatorsExecutor(ConstantinopleFixConfig, blockchain, Protocol.Ethash)
+  val istanbulValidators = ValidatorsExecutor(IstanbulConfig, blockchain, Protocol.Ethash)
+  val eip158ToByzantiumValidators = ValidatorsExecutor(Eip158ToByzantiumAt5Config, blockchain, Protocol.Ethash)
+  val byzantiumToConstantinopleAt5 = ValidatorsExecutor(ByzantiumToConstantinopleAt5, blockchain, Protocol.Ethash)
 }
 
 // Connected with: https://github.com/ethereum/tests/issues/480
-object ValidatorsWithSkippedPoW {
+class ValidatorsWithSkippedPoW(blockchain: Blockchain) {
 
   import BlockchainTestConfig._
 
-  val frontierValidators =  ValidatorsExecutor(FrontierConfig, new EthashTestBlockHeaderValidator(FrontierConfig))
-  val homesteadValidators = ValidatorsExecutor(HomesteadConfig, new EthashTestBlockHeaderValidator(HomesteadConfig))
-  val eip150Validators = ValidatorsExecutor(Eip150Config, new EthashTestBlockHeaderValidator(Eip150Config))
-  val frontierToHomesteadValidators = ValidatorsExecutor(FrontierToHomesteadAt5, new EthashTestBlockHeaderValidator(FrontierToHomesteadAt5))
-  val homesteadToEipValidators = ValidatorsExecutor(HomesteadToEIP150At5, new EthashTestBlockHeaderValidator(HomesteadToEIP150At5))
-  val homesteadToDaoValidators= ValidatorsExecutor(HomesteadToDaoAt5, new EthashTestBlockHeaderValidator(HomesteadToDaoAt5))
-  val eip158Validators = ValidatorsExecutor(Eip158Config, new EthashTestBlockHeaderValidator(Eip158Config))
-  val byzantiumValidators = ValidatorsExecutor(ByzantiumConfig, new EthashTestBlockHeaderValidator(ByzantiumConfig))
-  val constantinopleValidators = ValidatorsExecutor(ConstantinopleConfig, new EthashTestBlockHeaderValidator(ConstantinopleConfig))
-  val constantinopleFixValidators = ValidatorsExecutor(ConstantinopleFixConfig, new EthashTestBlockHeaderValidator(ConstantinopleFixConfig))
-  val istanbulValidators = ValidatorsExecutor(IstanbulConfig, new EthashTestBlockHeaderValidator(IstanbulConfig))
-  val eip158ToByzantiumValidators = ValidatorsExecutor(Eip158ToByzantiumAt5Config, new EthashTestBlockHeaderValidator(Eip158ToByzantiumAt5Config))
-  val byzantiumToConstantinopleAt5 = ValidatorsExecutor(ByzantiumToConstantinopleAt5, new EthashTestBlockHeaderValidator(ByzantiumToConstantinopleAt5))
+  val frontierValidators =  ValidatorsExecutor(FrontierConfig, new EthashTestBlockHeaderValidator(FrontierConfig, blockchain))
+  val homesteadValidators = ValidatorsExecutor(HomesteadConfig, new EthashTestBlockHeaderValidator(HomesteadConfig, blockchain))
+  val eip150Validators = ValidatorsExecutor(Eip150Config, new EthashTestBlockHeaderValidator(Eip150Config, blockchain))
+  val frontierToHomesteadValidators = ValidatorsExecutor(FrontierToHomesteadAt5, new EthashTestBlockHeaderValidator(FrontierToHomesteadAt5, blockchain))
+  val homesteadToEipValidators = ValidatorsExecutor(HomesteadToEIP150At5, new EthashTestBlockHeaderValidator(HomesteadToEIP150At5, blockchain))
+  val homesteadToDaoValidators= ValidatorsExecutor(HomesteadToDaoAt5, new EthashTestBlockHeaderValidator(HomesteadToDaoAt5, blockchain))
+  val eip158Validators = ValidatorsExecutor(Eip158Config, new EthashTestBlockHeaderValidator(Eip158Config, blockchain))
+  val byzantiumValidators = ValidatorsExecutor(ByzantiumConfig, new EthashTestBlockHeaderValidator(ByzantiumConfig, blockchain))
+  val constantinopleValidators = ValidatorsExecutor(ConstantinopleConfig, new EthashTestBlockHeaderValidator(ConstantinopleConfig, blockchain))
+  val constantinopleFixValidators = ValidatorsExecutor(ConstantinopleFixConfig, new EthashTestBlockHeaderValidator(ConstantinopleFixConfig, blockchain))
+  val istanbulValidators = ValidatorsExecutor(IstanbulConfig, new EthashTestBlockHeaderValidator(IstanbulConfig, blockchain))
+  val eip158ToByzantiumValidators = ValidatorsExecutor(Eip158ToByzantiumAt5Config, new EthashTestBlockHeaderValidator(Eip158ToByzantiumAt5Config, blockchain))
+  val byzantiumToConstantinopleAt5 = ValidatorsExecutor(ByzantiumToConstantinopleAt5, new EthashTestBlockHeaderValidator(ByzantiumToConstantinopleAt5, blockchain))
 }

src/ets/scala/io/iohk/ethereum/ets/blockchain/EthashTestBlockHeaderValidator.scala

@@ -3,18 +3,18 @@ package io.iohk.ethereum.ets.blockchain
 import io.iohk.ethereum.consensus.difficulty.DifficultyCalculator
 import io.iohk.ethereum.consensus.ethash.difficulty.EthashDifficultyCalculator
 import io.iohk.ethereum.consensus.ethash.validators.EthashBlockHeaderValidator
-import io.iohk.ethereum.consensus.validators.{ BlockHeaderError, BlockHeaderValid, BlockHeaderValidatorSkeleton }
-import io.iohk.ethereum.domain.BlockHeader
+import io.iohk.ethereum.consensus.validators.{BlockHeaderError, BlockHeaderValid, BlockHeaderValidatorSkeleton}
+import io.iohk.ethereum.domain.{BlockHeader, Blockchain}
 import io.iohk.ethereum.utils.BlockchainConfig
 
-class EthashTestBlockHeaderValidator(blockchainConfig: BlockchainConfig) extends BlockHeaderValidatorSkeleton(blockchainConfig) {
+class EthashTestBlockHeaderValidator(blockchainConfig: BlockchainConfig, blockchain: Blockchain) extends BlockHeaderValidatorSkeleton(blockchainConfig) {
   import EthashBlockHeaderValidator._
 
   // NOTE the below comment is from before PoW decoupling
   // we need concurrent map since validators can be used from multiple places
   protected val powCaches: java.util.concurrent.ConcurrentMap[Long, PowCacheData] = new java.util.concurrent.ConcurrentHashMap[Long, PowCacheData]()
 
-  protected def difficulty: DifficultyCalculator = new EthashDifficultyCalculator(blockchainConfig)
+  protected def difficulty: DifficultyCalculator = new EthashDifficultyCalculator(blockchainConfig, EthashDifficultyCalculator.grandparentsDataGetterFromBlockchain(blockchain))
 
   def validateEvenMore(blockHeader: BlockHeader, parentHeader: BlockHeader): Either[BlockHeaderError, BlockHeaderValid] =
     Right(BlockHeaderValid)

src/ets/scala/io/iohk/ethereum/ets/blockchain/ScenarioSetup.scala

@@ -52,11 +52,14 @@ abstract class ScenarioSetup(_vm: VMImpl, scenario: BlockchainScenario) {
   // according to: https://github.com/ethereum/tests/issues/480 only "NoProof" value should change our current implementation
   def shouldSkipPoW: Boolean = scenario.sealEngine.contains("NoProof")
 
-  val (blockchainConfig, validators) = buildBlockchainConfig(scenario.network, shouldSkipPoW)
-
-  //val validators = StdEthashValidators(blockchainConfig)
   val blockchain: BlockchainImpl = ScenarioSetup.getBlockchain
 
+  val validatorsWithPow = new Validators(blockchain)
+
+  val validatorsWithSkippedPoW = new ValidatorsWithSkippedPoW(blockchain)
+
+  val (blockchainConfig, validators) = buildBlockchainConfig(scenario.network, shouldSkipPoW)
+
   val consensus: TestConsensus = ScenarioSetup.loadEthashConsensus(_vm, blockchain, blockchainConfig, validators)
 
   val emptyWorld: InMemoryWorldStateProxy =
@@ -118,41 +121,41 @@ abstract class ScenarioSetup(_vm: VMImpl, scenario: BlockchainScenario) {
   }
 
   private def baseBlockchainConfig(network: String): (BlockchainConfig, ValidatorsExecutor) = network match {
-    case "EIP150" => (Eip150Config, Validators.eip150Validators)
-    case "Frontier" => (FrontierConfig, Validators.frontierValidators)
-    case "Homestead" => (HomesteadConfig, Validators.homesteadValidators)
-    case "FrontierToHomesteadAt5" => (FrontierToHomesteadAt5, Validators.frontierToHomesteadValidators)
-    case "HomesteadToEIP150At5" => (HomesteadToEIP150At5, Validators.homesteadToEipValidators)
-    case "EIP158" => (Eip158Config, Validators.eip158Validators)
-    case "HomesteadToDaoAt5" => (HomesteadToDaoAt5, Validators.homesteadToDaoValidators)
-    case "Byzantium" => (ByzantiumConfig, Validators.byzantiumValidators)
-    case "Constantinople" => (ConstantinopleConfig, Validators.constantinopleValidators)
-    case "EIP158ToByzantiumAt5" => (Eip158ToByzantiumAt5Config, Validators.eip158ToByzantiumValidators)
-    case "ByzantiumToConstantinopleFixAt5" => (ByzantiumToConstantinopleAt5, Validators.byzantiumToConstantinopleAt5)
-    case "ConstantinopleFix" => (ConstantinopleFixConfig, Validators.constantinopleValidators)
-    case "Istanbul" => (IstanbulConfig, Validators.istanbulValidators)
+    case "EIP150" => (Eip150Config, validatorsWithPow.eip150Validators)
+    case "Frontier" => (FrontierConfig, validatorsWithPow.frontierValidators)
+    case "Homestead" => (HomesteadConfig, validatorsWithPow.homesteadValidators)
+    case "FrontierToHomesteadAt5" => (FrontierToHomesteadAt5, validatorsWithPow.frontierToHomesteadValidators)
+    case "HomesteadToEIP150At5" => (HomesteadToEIP150At5, validatorsWithPow.homesteadToEipValidators)
+    case "EIP158" => (Eip158Config, validatorsWithPow.eip158Validators)
+    case "HomesteadToDaoAt5" => (HomesteadToDaoAt5, validatorsWithPow.homesteadToDaoValidators)
+    case "Byzantium" => (ByzantiumConfig, validatorsWithPow.byzantiumValidators)
+    case "Constantinople" => (ConstantinopleConfig, validatorsWithPow.constantinopleValidators)
+    case "EIP158ToByzantiumAt5" => (Eip158ToByzantiumAt5Config, validatorsWithPow.eip158ToByzantiumValidators)
+    case "ByzantiumToConstantinopleFixAt5" => (ByzantiumToConstantinopleAt5, validatorsWithPow.byzantiumToConstantinopleAt5)
+    case "ConstantinopleFix" => (ConstantinopleFixConfig, validatorsWithPow.constantinopleValidators)
+    case "Istanbul" => (IstanbulConfig, validatorsWithPow.istanbulValidators)
     // Some default config, test will fail or be canceled
-    case _ => (FrontierConfig, Validators.frontierValidators)
+    case _ => (FrontierConfig, validatorsWithPow.frontierValidators)
   }
 
   private def withSkippedPoWValidationBlockchainConfig(network: String): (BlockchainConfig, ValidatorsExecutor) =
     network match {
-      case "EIP150" => (Eip150Config, ValidatorsWithSkippedPoW.eip150Validators)
-      case "Frontier" => (FrontierConfig, ValidatorsWithSkippedPoW.frontierValidators)
-      case "Homestead" => (HomesteadConfig, ValidatorsWithSkippedPoW.homesteadValidators)
-      case "FrontierToHomesteadAt5" => (FrontierToHomesteadAt5, ValidatorsWithSkippedPoW.frontierToHomesteadValidators)
-      case "HomesteadToEIP150At5" => (HomesteadToEIP150At5, ValidatorsWithSkippedPoW.homesteadToEipValidators)
-      case "EIP158" => (Eip158Config, ValidatorsWithSkippedPoW.eip158Validators)
-      case "HomesteadToDaoAt5" => (HomesteadToDaoAt5, ValidatorsWithSkippedPoW.homesteadToDaoValidators)
-      case "Byzantium" => (ByzantiumConfig, ValidatorsWithSkippedPoW.byzantiumValidators)
-      case "Constantinople" => (ConstantinopleConfig, ValidatorsWithSkippedPoW.constantinopleValidators)
-      case "EIP158ToByzantiumAt5" => (Eip158ToByzantiumAt5Config, ValidatorsWithSkippedPoW.eip158ToByzantiumValidators)
+      case "EIP150" => (Eip150Config, validatorsWithSkippedPoW.eip150Validators)
+      case "Frontier" => (FrontierConfig, validatorsWithSkippedPoW.frontierValidators)
+      case "Homestead" => (HomesteadConfig, validatorsWithSkippedPoW.homesteadValidators)
+      case "FrontierToHomesteadAt5" => (FrontierToHomesteadAt5, validatorsWithSkippedPoW.frontierToHomesteadValidators)
+      case "HomesteadToEIP150At5" => (HomesteadToEIP150At5, validatorsWithSkippedPoW.homesteadToEipValidators)
+      case "EIP158" => (Eip158Config, validatorsWithSkippedPoW.eip158Validators)
+      case "HomesteadToDaoAt5" => (HomesteadToDaoAt5, validatorsWithSkippedPoW.homesteadToDaoValidators)
+      case "Byzantium" => (ByzantiumConfig, validatorsWithSkippedPoW.byzantiumValidators)
+      case "Constantinople" => (ConstantinopleConfig, validatorsWithSkippedPoW.constantinopleValidators)
+      case "EIP158ToByzantiumAt5" => (Eip158ToByzantiumAt5Config, validatorsWithSkippedPoW.eip158ToByzantiumValidators)
       case "ByzantiumToConstantinopleFixAt5" =>
-        (ByzantiumToConstantinopleAt5, ValidatorsWithSkippedPoW.byzantiumToConstantinopleAt5)
-      case "ConstantinopleFix" => (ConstantinopleFixConfig, ValidatorsWithSkippedPoW.constantinopleFixValidators)
-      case "Istanbul" => (IstanbulConfig, ValidatorsWithSkippedPoW.istanbulValidators)
+        (ByzantiumToConstantinopleAt5, validatorsWithSkippedPoW.byzantiumToConstantinopleAt5)
+      case "ConstantinopleFix" => (ConstantinopleFixConfig, validatorsWithSkippedPoW.constantinopleFixValidators)
+      case "Istanbul" => (IstanbulConfig, validatorsWithSkippedPoW.istanbulValidators)
       // Some default config, test will fail or be canceled
-      case _ => (FrontierConfig, ValidatorsWithSkippedPoW.frontierValidators)
+      case _ => (FrontierConfig, validatorsWithSkippedPoW.frontierValidators)
     }
 
   private def decode(s: String): Array[Byte] = {

src/main/scala/io/iohk/ethereum/consensus/ConsensusBuilder.scala

@@ -27,7 +27,7 @@ trait StdConsensusBuilder extends ConsensusBuilder {
   protected def buildEthashConsensus(): ethash.EthashConsensus = {
     val specificConfig = ethash.EthashConfig(mantisConfig)
     val fullConfig = newConfig(specificConfig)
-    val validators = ValidatorsExecutor(blockchainConfig, consensusConfig.protocol)
+    val validators = ValidatorsExecutor(blockchainConfig, blockchain, consensusConfig.protocol)
     val consensus = EthashConsensus(vm, blockchain, blockchainConfig, fullConfig, validators)
     consensus
   }

src/main/scala/io/iohk/ethereum/consensus/blocks/CheckpointBlockGenerator.scala

@@ -10,14 +10,14 @@ class CheckpointBlockGenerator {
   def generate(parent: Block, checkpoint: Checkpoint): Block = {
     val blockNumber = parent.number + 1
     // we are using a predictable value for timestamp so that each federation node generates identical block
-    // see ETCM-173
     val timestamp = parent.header.unixTimestamp + 1
 
     val header = BlockHeader(
       parentHash = parent.hash,
       ommersHash = BlockHeader.EmptyOmmers,
       beneficiary = BlockHeader.EmptyBeneficiary,
-      difficulty = parent.header.difficulty,
+      // there is no PoW here
+      difficulty = BlockHeader.EmptyDifficulty,
       number = blockNumber,
       gasLimit = parent.header.gasLimit,
       unixTimestamp = timestamp,

src/main/scala/io/iohk/ethereum/consensus/ethash/EthashConsensus.scala

@@ -178,7 +178,10 @@ object EthashConsensus {
       validators: ValidatorsExecutor
   ): EthashConsensus = {
 
-    val difficultyCalculator = new EthashDifficultyCalculator(blockchainConfig)
+    val difficultyCalculator = new EthashDifficultyCalculator(
+      blockchainConfig,
+      EthashDifficultyCalculator.grandparentsDataGetterFromBlockchain(blockchain)
+    )
 
     val blockPreparator = new BlockPreparator(
       vm = vm,

src/main/scala/io/iohk/ethereum/consensus/ethash/difficulty/EthashDifficultyCalculator.scala

@@ -1,10 +1,13 @@
 package io.iohk.ethereum.consensus.ethash.difficulty
 
+import akka.util.ByteString
 import io.iohk.ethereum.consensus.difficulty.DifficultyCalculator
-import io.iohk.ethereum.domain.BlockHeader
-import io.iohk.ethereum.utils.BlockchainConfig
+import io.iohk.ethereum.consensus.ethash.difficulty.EthashDifficultyCalculator._
+import io.iohk.ethereum.domain.{BlockHeader, Blockchain}
+import io.iohk.ethereum.utils.{BlockchainConfig, ByteStringUtils}
 
-class EthashDifficultyCalculator(blockchainConfig: BlockchainConfig) extends DifficultyCalculator {
+class EthashDifficultyCalculator(blockchainConfig: BlockchainConfig, grandParentsDataGetter: GrandparentsDataGetter)
+    extends DifficultyCalculator {
   import blockchainConfig._
 
   val DifficultyBoundDivision: Int = 2048
@@ -15,9 +18,15 @@ class EthashDifficultyCalculator(blockchainConfig: BlockchainConfig) extends Dif
   val ConstantinopleRelaxDifficulty: BigInt = 5000000
 
   def calculateDifficulty(blockNumber: BigInt, blockTimestamp: Long, parentHeader: BlockHeader): BigInt = {
-    lazy val timestampDiff = blockTimestamp - parentHeader.unixTimestamp
+    lazy val GrandparentsData(grandparentDifficulty, grandparentsTimestampDiff) = grandParentsDataGetter(
+      parentHeader.hash
+    )
+    val parentHeaderDifficulty = if (!parentHeader.hasCheckpoint) parentHeader.difficulty else grandparentDifficulty
+    lazy val timestampDiff = if (!parentHeader.hasCheckpoint) {
+      blockTimestamp - parentHeader.unixTimestamp
+    } else grandparentsTimestampDiff
 
-    val x: BigInt = parentHeader.difficulty / DifficultyBoundDivision
+    val x: BigInt = parentHeaderDifficulty / DifficultyBoundDivision
     val c: BigInt =
       if (blockNumber < homesteadBlockNumber) {
         if (blockTimestamp < parentHeader.unixTimestamp + 13) 1 else -1
@@ -45,7 +54,7 @@ class EthashDifficultyCalculator(blockchainConfig: BlockchainConfig) extends Dif
       } else
         0
 
-    val difficultyWithoutBomb = MinimumDifficulty.max(parentHeader.difficulty + x * c)
+    val difficultyWithoutBomb = MinimumDifficulty.max(parentHeaderDifficulty + x * c)
     difficultyWithoutBomb + extraDifficulty
   }
 
@@ -60,3 +69,21 @@ class EthashDifficultyCalculator(blockchainConfig: BlockchainConfig) extends Dif
     }
   }
 }
+
+object EthashDifficultyCalculator {
+  case class GrandparentsData(grandparentDifficulty: BigInt, grandparentsTimestampDiff: Long)
+  type GrandparentsDataGetter = ByteString => GrandparentsData
+
+  def grandparentsDataGetterFromBlockchain(blockchain: Blockchain): GrandparentsDataGetter = { hash =>
+    // not having parent/grandparent headers means that sync is corrupted and we want to fail fast
+    (for {
+      parent <- blockchain.getBlockHeaderByHash(hash)
+      grandparent <- blockchain.getBlockHeaderByHash(parent.hash)
+    } yield GrandparentsData(parent.difficulty, parent.unixTimestamp - grandparent.unixTimestamp))
+      .getOrElse(
+        throw new RuntimeException(
+          s"Blockchain is corrupted - missing parent/grandparent headers for hash ${ByteStringUtils.hash2string(hash)}"
+        )
+      )
+  }
+}