Skip to content

Commit

Permalink
[ETCM-79] checkpointing QA JRC
Browse files Browse the repository at this point in the history
  • Loading branch information
@rtkaczyk
rtkaczyk committed Oct 20, 2020
1 parent effa2d1 commit 49b8a81
Show file tree
Hide file tree
Showing 12 changed files with 514 additions and 50 deletions.
98 changes: 97 additions & 1 deletion insomnia_workspace.json
View file
Expand Up @@ -139,6 +139,102 @@
"metaSortKey": -1600249374160,
"_type": "request_group"
},
{
"_id": "req_3ae9151dec5b4046b06fdb3408e9ab1f",
"authentication": {},
"body": {
"mimeType": "application/json",
"text": "{\n\t\"jsonrpc\": \"2.0\",\n\t\"id\": 1,\n\t\"method\": \"qa_generateCheckpoint\",\n\t\"params\": [\t\t[\"0x926db397ed35bedee660fe5bf6879679fa71f1fe8c27823f7f6a1e5d96a49b94\"], \"0xe160bdd7664e1b921612a4ed225321bdf3b8f70beb6b968c7a423d6022e39e16\"\n\t]\n}"
},
"created": 1572007151716,
"description": "",
"headers": [
{
"id": "pair_6f2ea2de85ee4eabbbd25bde888e9dc1",
"name": "Content-Type",
"value": "application/json"
}
],
"isPrivate": false,
"metaSortKey": -1566535778023.25,
"method": "POST",
"modified": 1572263390995,
"name": "qa_generateCheckpoint",
"parameters": [],
"parentId": "fld_2b54cbb84e244284b3ef752c5f805376",
"settingDisableRenderRequestBody": false,
"settingEncodeUrl": true,
"settingFollowRedirects": "global",
"settingRebuildPath": true,
"settingSendCookies": true,
"settingStoreCookies": true,
"url": "{{ node_url }}",
"_type": "request"
},
{
"_id": "req_e1287e4fcba348eea9d326bb208092c3",
"authentication": {},
"body": {
"mimeType": "application/json",
"text": "{\n\t\"jsonrpc\": \"2.0\",\n\t\"id\": 1,\n\t\"method\": \"qa_generateCheckpoint\",\n\t\"params\": [\t\t[\"0x926db397ed35bedee660fe5bf6879679fa71f1fe8c27823f7f6a1e5d96a49b94\"]\n\t]\n}"
},
"created": 1573216065706,
"description": "",
"headers": [
{
"id": "pair_6f2ea2de85ee4eabbbd25bde888e9dc1",
"name": "Content-Type",
"value": "application/json"
}
],
"isPrivate": false,
"metaSortKey": -1566394039140.875,
"method": "POST",
"modified": 1573216279799,
"name": "qa_generateCheckpoint (for latest block)",
"parameters": [],
"parentId": "fld_2b54cbb84e244284b3ef752c5f805376",
"settingDisableRenderRequestBody": false,
"settingEncodeUrl": true,
"settingFollowRedirects": "global",
"settingRebuildPath": true,
"settingSendCookies": true,
"settingStoreCookies": true,
"url": "{{ node_url }}",
"_type": "request"
},
{
"_id": "req_f2986c964bf74360878144213ca342a7",
"authentication": {},
"body": {
"mimeType": "application/json",
"text": "{\n\t\"jsonrpc\": \"2.0\",\n\t\"id\": 1,\n\t\"method\": \"qa_getFederationMembersInfo\",\n\t\"params\": []\n}"
},
"created": 1573042743181,
"description": "",
"headers": [
{
"id": "pair_ba5a5914f02a4b27b86a330054582828",
"name": "Content-Type",
"value": "application/json"
}
],
"isPrivate": false,
"metaSortKey": -1568046037126,
"method": "POST",
"modified": 1573042783130,
"name": "qa_getFederationMembersInfo",
"parameters": [],
"parentId": "fld_2b54cbb84e244284b3ef752c5f805376",
"settingDisableRenderRequestBody": false,
"settingEncodeUrl": true,
"settingFollowRedirects": "global",
"settingRebuildPath": true,
"settingSendCookies": true,
"settingStoreCookies": true,
"url": "{{ node_url }}",
"_type": "request"
},
{
"_id": "req_dd6e5c718f974407bb79fe3c953b7106",
"parentId": "fld_72829b866f0441e184e0d1a2030f8220",
Expand Down Expand Up @@ -1124,4 +1220,4 @@
"_type": "environment"
}
]
}
}
68 changes: 41 additions & 27 deletions src/ets/scala/io/iohk/ethereum/ets/blockchain/ScenarioSetup.scala
View file
Expand Up @@ -23,9 +23,14 @@ import scala.util.{Failure, Success, Try}
object ScenarioSetup {
val testContext = ExecutionContext.fromExecutor(Executors.newFixedThreadPool(4))
val specificConfig = ethash.EthashConfig(Config.config)
val fullConfig = FullConsensusConfig(ConsensusConfig(Config.config)(null), specificConfig)

def loadEthashConsensus(vm: VMImpl, blockchain: BlockchainImpl, blockchainConfig: BlockchainConfig, validators: ValidatorsExecutor): ethash.EthashConsensus = {
val fullConfig = FullConsensusConfig(ConsensusConfig(Config.config), specificConfig)

def loadEthashConsensus(
vm: VMImpl,
blockchain: BlockchainImpl,
blockchainConfig: BlockchainConfig,
validators: ValidatorsExecutor
): ethash.EthashConsensus = {
val consensus = EthashConsensus(vm, blockchain, blockchainConfig, fullConfig, validators)
consensus
}
Expand All @@ -34,7 +39,6 @@ object ScenarioSetup {
override val pruningMode: PruningMode = ArchivePruning
}


def getBlockchain: BlockchainImpl = {
val storagesInstance = new EphemDataSourceComponent with Pruning with Storages.DefaultStorages
BlockchainImpl(storagesInstance.storages)
Expand All @@ -61,21 +65,28 @@ abstract class ScenarioSetup(_vm: VMImpl, scenario: BlockchainScenario) {
val ledger =
new LedgerImpl(
blockchain,
new BlockQueue(blockchain, 50, 50), blockchainConfig, consensus, ScenarioSetup.testContext)
new BlockQueue(blockchain, 50, 50),
blockchainConfig,
consensus,
ScenarioSetup.testContext
)

def loadGenesis(): Block = {
val genesisBlock = scenario.genesisRLP match {
case Some(rlp) =>
val block = rlp.toArray.toBlock
assert(block.header == scenario.genesisBlockHeader.toBlockHeader,
"decoded genesis block header did not match the expectation")
assert(
block.header == scenario.genesisBlockHeader.toBlockHeader,
"decoded genesis block header did not match the expectation"
)
block

case None =>
Block(scenario.genesisBlockHeader.toBlockHeader, BlockBody(Nil, Nil))
}

blockchain.storeBlock(genesisBlock)
blockchain
.storeBlock(genesisBlock)
.and(blockchain.storeReceipts(genesisBlock.header.hash, Nil))
.and(blockchain.storeTotalDifficulty(genesisBlock.header.hash, genesisBlock.header.difficulty))
.commit()
Expand All @@ -85,7 +96,8 @@ abstract class ScenarioSetup(_vm: VMImpl, scenario: BlockchainScenario) {

val initialWorld: InMemoryWorldStateProxy = InMemoryWorldStateProxy.persistState(getWorldState(scenario.pre))

val finalWorld: Option[InMemoryWorldStateProxy] = scenario.postState.map(postState => InMemoryWorldStateProxy.persistState(getWorldState(postState)))
val finalWorld: Option[InMemoryWorldStateProxy] =
scenario.postState.map(postState => InMemoryWorldStateProxy.persistState(getWorldState(postState)))

def getBestBlock: Option[Block] = {
val bestBlockNumber = blockchain.getBestBlockNumber()
Expand Down Expand Up @@ -123,23 +135,25 @@ abstract class ScenarioSetup(_vm: VMImpl, scenario: BlockchainScenario) {
case _ => (FrontierConfig, Validators.frontierValidators)
}

private def withSkippedPoWValidationBlockchainConfig(network: String): (BlockchainConfig, ValidatorsExecutor) = network match {
case "EIP150" => (Eip150Config, ValidatorsWithSkippedPoW.eip150Validators)
case "Frontier" => (FrontierConfig, ValidatorsWithSkippedPoW.frontierValidators)
case "Homestead" => (HomesteadConfig, ValidatorsWithSkippedPoW.homesteadValidators)
case "FrontierToHomesteadAt5" => (FrontierToHomesteadAt5, ValidatorsWithSkippedPoW.frontierToHomesteadValidators)
case "HomesteadToEIP150At5" => (HomesteadToEIP150At5, ValidatorsWithSkippedPoW.homesteadToEipValidators)
case "EIP158" => (Eip158Config, ValidatorsWithSkippedPoW.eip158Validators)
case "HomesteadToDaoAt5" => (HomesteadToDaoAt5, ValidatorsWithSkippedPoW.homesteadToDaoValidators)
case "Byzantium" => (ByzantiumConfig, ValidatorsWithSkippedPoW.byzantiumValidators)
case "Constantinople" => (ConstantinopleConfig, ValidatorsWithSkippedPoW.constantinopleValidators)
case "EIP158ToByzantiumAt5" => (Eip158ToByzantiumAt5Config, ValidatorsWithSkippedPoW.eip158ToByzantiumValidators)
case "ByzantiumToConstantinopleFixAt5" => (ByzantiumToConstantinopleAt5, ValidatorsWithSkippedPoW.byzantiumToConstantinopleAt5)
case "ConstantinopleFix" => (ConstantinopleFixConfig, ValidatorsWithSkippedPoW.constantinopleFixValidators)
case "Istanbul" => (IstanbulConfig, ValidatorsWithSkippedPoW.istanbulValidators)
// Some default config, test will fail or be canceled
case _ => (FrontierConfig, ValidatorsWithSkippedPoW.frontierValidators)
}
private def withSkippedPoWValidationBlockchainConfig(network: String): (BlockchainConfig, ValidatorsExecutor) =
network match {
case "EIP150" => (Eip150Config, ValidatorsWithSkippedPoW.eip150Validators)
case "Frontier" => (FrontierConfig, ValidatorsWithSkippedPoW.frontierValidators)
case "Homestead" => (HomesteadConfig, ValidatorsWithSkippedPoW.homesteadValidators)
case "FrontierToHomesteadAt5" => (FrontierToHomesteadAt5, ValidatorsWithSkippedPoW.frontierToHomesteadValidators)
case "HomesteadToEIP150At5" => (HomesteadToEIP150At5, ValidatorsWithSkippedPoW.homesteadToEipValidators)
case "EIP158" => (Eip158Config, ValidatorsWithSkippedPoW.eip158Validators)
case "HomesteadToDaoAt5" => (HomesteadToDaoAt5, ValidatorsWithSkippedPoW.homesteadToDaoValidators)
case "Byzantium" => (ByzantiumConfig, ValidatorsWithSkippedPoW.byzantiumValidators)
case "Constantinople" => (ConstantinopleConfig, ValidatorsWithSkippedPoW.constantinopleValidators)
case "EIP158ToByzantiumAt5" => (Eip158ToByzantiumAt5Config, ValidatorsWithSkippedPoW.eip158ToByzantiumValidators)
case "ByzantiumToConstantinopleFixAt5" =>
(ByzantiumToConstantinopleAt5, ValidatorsWithSkippedPoW.byzantiumToConstantinopleAt5)
case "ConstantinopleFix" => (ConstantinopleFixConfig, ValidatorsWithSkippedPoW.constantinopleFixValidators)
case "Istanbul" => (IstanbulConfig, ValidatorsWithSkippedPoW.istanbulValidators)
// Some default config, test will fail or be canceled
case _ => (FrontierConfig, ValidatorsWithSkippedPoW.frontierValidators)
}

private def decode(s: String): Array[Byte] = {
val stripped = s.replaceFirst("^0x", "")
Expand All @@ -152,7 +166,7 @@ abstract class ScenarioSetup(_vm: VMImpl, scenario: BlockchainScenario) {
case Success(block) =>
Some(block)

case Failure(ex) =>
case Failure(ex) =>
ex.printStackTrace()
None
}
Expand Down
3 changes: 1 addition & 2 deletions src/main/scala/io/iohk/ethereum/consensus/ConsensusConfig.scala
View file
Expand Up @@ -4,7 +4,6 @@ import akka.util.ByteString
import com.typesafe.config.{Config => TypesafeConfig}
import io.iohk.ethereum.consensus.validators.BlockHeaderValidator
import io.iohk.ethereum.domain.Address
import io.iohk.ethereum.nodebuilder.ShutdownHookBuilder
import io.iohk.ethereum.utils.Logger

/**
Expand Down Expand Up @@ -59,7 +58,7 @@ object ConsensusConfig extends Logger {
Protocol(protocol)
}

def apply(mantisConfig: TypesafeConfig)(shutdownHook: ShutdownHookBuilder): ConsensusConfig = {
def apply(mantisConfig: TypesafeConfig): ConsensusConfig = {
val config = mantisConfig.getConfig(Keys.Consensus)

val protocol = readProtocol(config)
Expand Down
5 changes: 2 additions & 3 deletions src/main/scala/io/iohk/ethereum/consensus/ConsensusConfigBuilder.scala
View file
@@ -1,10 +1,9 @@
package io.iohk.ethereum.consensus

import io.iohk.ethereum.nodebuilder.ShutdownHookBuilder
import io.iohk.ethereum.utils.Config

trait ConsensusConfigBuilder { self: ShutdownHookBuilder =>
protected def buildConsensusConfig(): ConsensusConfig = ConsensusConfig(Config.config)(this)
trait ConsensusConfigBuilder {
protected def buildConsensusConfig(): ConsensusConfig = ConsensusConfig(Config.config)

lazy val consensusConfig: ConsensusConfig = buildConsensusConfig()
}
3 changes: 3 additions & 0 deletions src/main/scala/io/iohk/ethereum/crypto/ECDSASignature.scala
View file
Expand Up @@ -37,6 +37,9 @@ object ECDSASignature {
None
}

def sign(message: ByteString, prvKey: ByteString): ECDSASignature =
sign(message.toArray, keyPairFromPrvKey(prvKey.toArray), None)

def sign(message: Array[Byte], keyPair: AsymmetricCipherKeyPair, chainId: Option[Byte] = None): ECDSASignature = {
val signer = new ECDSASigner(new HMacDSAKCalculator(new SHA256Digest))
signer.init(true, keyPair.getPrivate)
Expand Down
5 changes: 5 additions & 0 deletions src/main/scala/io/iohk/ethereum/crypto/package.scala
View file
Expand Up @@ -79,6 +79,11 @@ package object crypto {
keyPairFromPrvKey(privateKey)
}

def keyPairFromPrvKey(prvKeyBytes: ByteString): AsymmetricCipherKeyPair = {
val privateKey = BigInt(1, prvKeyBytes.toArray)
keyPairFromPrvKey(privateKey)
}

def keyPairFromPrvKey(prvKey: BigInt): AsymmetricCipherKeyPair = {
val publicKey = curve.getG.multiply(prvKey.bigInteger).normalize()
new AsymmetricCipherKeyPair(
Expand Down
12 changes: 12 additions & 0 deletions src/main/scala/io/iohk/ethereum/jsonrpc/JsonRpcController.scala
View file
Expand Up @@ -11,6 +11,12 @@ import org.json4s.JsonDSL._
import com.typesafe.config.{Config => TypesafeConfig}
import io.iohk.ethereum.jsonrpc.DebugService.{ListPeersInfoRequest, ListPeersInfoResponse}
import io.iohk.ethereum.jsonrpc.JsonRpcErrors.InvalidParams
import io.iohk.ethereum.jsonrpc.QAService.{
GenerateCheckpointRequest,
GenerateCheckpointResponse,
GetFederationMembersInfoRequest,
GetFederationMembersInfoResponse
}
import io.iohk.ethereum.jsonrpc.TestService._
import io.iohk.ethereum.jsonrpc.server.http.JsonRpcHttpServer.JsonRpcHttpServerConfig
import io.iohk.ethereum.jsonrpc.server.ipc.JsonRpcIpcServer.JsonRpcIpcServerConfig
Expand Down Expand Up @@ -350,6 +356,12 @@ class JsonRpcController(
private def handleQARequest: PartialFunction[JsonRpcRequest, Future[JsonRpcResponse]] = {
case req @ JsonRpcRequest(_, "qa_mineBlocks", _, _) =>
handle[QAService.MineBlocksRequest, QAService.MineBlocksResponse](qaService.mineBlocks, req)

case req @ JsonRpcRequest(_, "qa_generateCheckpoint", _, _) =>
handle[GenerateCheckpointRequest, GenerateCheckpointResponse](qaService.generateCheckpoint, req)

case req @ JsonRpcRequest(_, "qa_getFederationMembersInfo", _, _) =>
handle[GetFederationMembersInfoRequest, GetFederationMembersInfoResponse](qaService.getFederationMembersInfo, req)
}

private def handleCheckpointingRequest: PartialFunction[JsonRpcRequest, Future[JsonRpcResponse]] = {
Expand Down
39 changes: 37 additions & 2 deletions src/main/scala/io/iohk/ethereum/jsonrpc/QAJsonMethodsImplicits.scala
View file
@@ -1,8 +1,11 @@
package io.iohk.ethereum.jsonrpc

import io.iohk.ethereum.jsonrpc.JsonRpcController.Codec
import akka.util.ByteString
import io.iohk.ethereum.jsonrpc.JsonRpcController.JsonDecoder.NoParamsDecoder
import io.iohk.ethereum.jsonrpc.JsonRpcController.{Codec, JsonEncoder}
import io.iohk.ethereum.jsonrpc.JsonRpcErrors.InvalidParams
import io.iohk.ethereum.jsonrpc.QAService.{MineBlocksRequest, MineBlocksResponse}
import io.iohk.ethereum.jsonrpc.QAService._
import org.json4s.Extraction
import org.json4s.JsonAST._

object QAJsonMethodsImplicits extends JsonMethodsImplicits {
Expand All @@ -29,4 +32,36 @@ object QAJsonMethodsImplicits extends JsonMethodsImplicits {
"message" -> t.message.fold[JValue](JNull)(JString)
)
}

implicit val qa_generateCheckpoint: Codec[GenerateCheckpointRequest, GenerateCheckpointResponse] =
new Codec[GenerateCheckpointRequest, GenerateCheckpointResponse] {
def decodeJson(params: Option[JArray]): Either[JsonRpcError, GenerateCheckpointRequest] = {
params match {
case Some(JArray((keys: JArray) :: JString(hash) :: Nil)) =>
for {
blockHash <- extractHash(hash)
keys <- parseKeysList(keys)
} yield GenerateCheckpointRequest(keys, Some(blockHash))
case Some(JArray((keys: JArray) :: Nil)) =>
parseKeysList(keys).map(GenerateCheckpointRequest(_, None))
case _ =>
Left(InvalidParams())
}
}

def encodeJson(t: GenerateCheckpointResponse): JValue = Extraction.decompose(t.checkpoint)
}

private def parseKeysList(arr: JArray): Either[JsonRpcError, List[ByteString]] = {
import cats.implicits._
arr.arr.traverse {
case JString(key) => extractBytes(key)
case other => Left(InvalidParams(msg = s"Unable to parse private key, expected byte data but got: $other"))
}
}

implicit val qa_getFederationMembersInfo: Codec[GetFederationMembersInfoRequest, GetFederationMembersInfoResponse] =
new NoParamsDecoder(GetFederationMembersInfoRequest()) with JsonEncoder[GetFederationMembersInfoResponse] {
def encodeJson(t: GetFederationMembersInfoResponse): JValue = Extraction.decompose(t)
}
}

0 comments on commit 49b8a81

Please sign in to comment.