From 09df3f32b50a7bc9b2d79a4cfca021fd262fac06 Mon Sep 17 00:00:00 2001 From: tersec Date: Thu, 26 Oct 2023 03:40:04 +0000 Subject: [PATCH 01/35] add non-SZ getBlobSidecar and BlobSidecar database tests (#5528) --- AllTests-mainnet.md | 5 +- beacon_chain/beacon_chain_db.nim | 9 +- .../consensus_object_pools/blockchain_dag.nim | 13 --- tests/test_beacon_chain_db.nim | 97 ++++++++++++++++++- tests/testdbutil.nim | 5 + 5 files changed, 110 insertions(+), 19 deletions(-) diff --git a/AllTests-mainnet.md b/AllTests-mainnet.md index 48daf1f087..007d509785 100644 --- a/AllTests-mainnet.md +++ b/AllTests-mainnet.md @@ -48,6 +48,7 @@ OK: 3/3 Fail: 0/3 Skip: 0/3 + sanity check Deneb blocks [Preset: mainnet] OK + sanity check Deneb states [Preset: mainnet] OK + sanity check Deneb states, reusing buffers [Preset: mainnet] OK ++ sanity check blobs [Preset: mainnet] OK + sanity check genesis roundtrip [Preset: mainnet] OK + sanity check phase 0 blocks [Preset: mainnet] OK + sanity check phase 0 getState rollback [Preset: mainnet] OK @@ -55,7 +56,7 @@ OK: 3/3 Fail: 0/3 Skip: 0/3 + sanity check phase 0 states, reusing buffers [Preset: mainnet] OK + sanity check state diff roundtrip [Preset: mainnet] OK ``` -OK: 24/24 Fail: 0/24 Skip: 0/24 +OK: 25/25 Fail: 0/25 Skip: 0/25 ## Beacon state [Preset: mainnet] ```diff + Smoke test initialize_beacon_state_from_eth1 [Preset: mainnet] OK @@ -714,4 +715,4 @@ OK: 2/2 Fail: 0/2 Skip: 0/2 OK: 9/9 Fail: 0/9 Skip: 0/9 ---TOTAL--- -OK: 403/408 Fail: 0/408 Skip: 5/408 +OK: 404/409 Fail: 0/409 Skip: 5/409 diff --git a/beacon_chain/beacon_chain_db.nim b/beacon_chain/beacon_chain_db.nim index 0daea2fb36..2290505f51 100644 --- a/beacon_chain/beacon_chain_db.nim +++ b/beacon_chain/beacon_chain_db.nim @@ -797,7 +797,7 @@ proc putBlobSidecar*( proc delBlobSidecar*( db: BeaconChainDB, - root: Eth2Digest, index: BlobIndex) : bool = + root: Eth2Digest, index: BlobIndex): bool = db.blobs.del(blobkey(root, index)).expectDb() proc updateImmutableValidators*( @@ -1042,13 +1042,16 @@ proc getBlockSSZ*( getBlockSSZ(db, key, data, consensusFork.TrustedSignedBeaconBlock) proc getBlobSidecarSZ*(db: BeaconChainDB, root: Eth2Digest, index: BlobIndex, - data: var seq[byte]): - bool = + data: var seq[byte]): bool = let dataPtr = addr data # Short-lived func decode(data: openArray[byte]) = assign(dataPtr[], data) db.blobs.get(blobkey(root, index), decode).expectDb() +proc getBlobSidecar*(db: BeaconChainDB, root: Eth2Digest, index: BlobIndex, + value: var BlobSidecar): bool = + db.blobs.getSZSSZ(blobkey(root, index), value) == GetResult.found + proc getBlockSZ*( db: BeaconChainDB, key: Eth2Digest, data: var seq[byte], T: type phase0.TrustedSignedBeaconBlock): bool = diff --git a/beacon_chain/consensus_object_pools/blockchain_dag.nim b/beacon_chain/consensus_object_pools/blockchain_dag.nim index 9eb3d408fd..d601aa59f7 100644 --- a/beacon_chain/consensus_object_pools/blockchain_dag.nim +++ b/beacon_chain/consensus_object_pools/blockchain_dag.nim @@ -234,19 +234,6 @@ proc getForkedBlock*(db: BeaconChainDB, root: Eth2Digest): else: err() -proc containsBlock(dag: ChainDAGRef, bid: BlockId): bool = - let fork = dag.cfg.consensusForkAtEpoch(bid.slot.epoch) - if dag.db.containsBlock(bid.root, fork): - return true - - # TODO avoid loading bytes from era - var bytes: seq[byte] - (bid.slot <= dag.finalizedHead.slot and - getBlockSZ( - dag.era, getStateField(dag.headState, historical_roots).asSeq, - dag.headState.historical_summaries().asSeq, - bid.slot, bytes).isOk and bytes.len > 0) - proc getBlock*( dag: ChainDAGRef, bid: BlockId, T: type ForkyTrustedSignedBeaconBlock): Opt[T] = diff --git a/tests/test_beacon_chain_db.nim b/tests/test_beacon_chain_db.nim index 40ed628719..b9eee488a9 100644 --- a/tests/test_beacon_chain_db.nim +++ b/tests/test_beacon_chain_db.nim @@ -11,7 +11,6 @@ import unittest2, ../beacon_chain/beacon_chain_db, ../beacon_chain/spec/[beaconstate, forks, state_transition], - ../beacon_chain/spec/datatypes/[phase0, altair, bellatrix], ../beacon_chain/consensus_object_pools/blockchain_dag, eth/db/kvstore, # test utilies @@ -761,6 +760,102 @@ suite "Beacon chain DB" & preset(): check: hash_tree_root(state2[]) == root + test "sanity check blobs" & preset(): + const + blockRoot0 = Eth2Digest.fromHex( + "0x436f3180d2541a94560de3423d6d8f0e16e3b4c061ccd620bcfaee31424b5eb6") + blockRoot1 = Eth2Digest.fromHex( + "0x0d3730dc690d5fe4d39ded682d3aec7679af597d5fc6a65a7c8b4e20f228f96d") + + # Ensure minimal-difference pairs on both block root and blob index to + # verify that blobkey uses both + blobSidecar0 = BlobSidecar(block_root: blockRoot0, index: 3) + blobSidecar1 = BlobSidecar(block_root: blockRoot0, index: 2) + blobSidecar2 = BlobSidecar(block_root: blockRoot1, index: 2) + + let db = makeTestDB(SLOTS_PER_EPOCH) + + var + buf: seq[byte] + blobSidecar: BlobSidecar + + check: + not db.getBlobSidecar(blockRoot0, 3, blobSidecar) + not db.getBlobSidecar(blockRoot0, 2, blobSidecar) + not db.getBlobSidecar(blockRoot1, 2, blobSidecar) + not db.getBlobSidecarSZ(blockRoot0, 3, buf) + not db.getBlobSidecarSZ(blockRoot0, 2, buf) + not db.getBlobSidecarSZ(blockRoot1, 2, buf) + + db.putBlobSidecar(blobSidecar0) + + check: + db.getBlobSidecar(blockRoot0, 3, blobSidecar) + blobSidecar == blobSidecar0 + not db.getBlobSidecar(blockRoot0, 2, blobSidecar) + not db.getBlobSidecar(blockRoot1, 2, blobSidecar) + db.getBlobSidecarSZ(blockRoot0, 3, buf) + not db.getBlobSidecarSZ(blockRoot0, 2, buf) + not db.getBlobSidecarSZ(blockRoot1, 2, buf) + + db.putBlobSidecar(blobSidecar1) + + check: + db.getBlobSidecar(blockRoot0, 3, blobSidecar) + blobSidecar == blobSidecar0 + db.getBlobSidecar(blockRoot0, 2, blobSidecar) + blobSidecar == blobSidecar1 + not db.getBlobSidecar(blockRoot1, 2, blobSidecar) + db.getBlobSidecarSZ(blockRoot0, 3, buf) + db.getBlobSidecarSZ(blockRoot0, 2, buf) + not db.getBlobSidecarSZ(blockRoot1, 2, buf) + + check db.delBlobSidecar(blockRoot0, 3) + + check: + not db.getBlobSidecar(blockRoot0, 3, blobSidecar) + db.getBlobSidecar(blockRoot0, 2, blobSidecar) + blobSidecar == blobSidecar1 + not db.getBlobSidecar(blockRoot1, 2, blobSidecar) + not db.getBlobSidecarSZ(blockRoot0, 3, buf) + db.getBlobSidecarSZ(blockRoot0, 2, buf) + not db.getBlobSidecarSZ(blockRoot1, 2, buf) + + db.putBlobSidecar(blobSidecar2) + + check: + not db.getBlobSidecar(blockRoot0, 3, blobSidecar) + db.getBlobSidecar(blockRoot0, 2, blobSidecar) + blobSidecar == blobSidecar1 + db.getBlobSidecar(blockRoot1, 2, blobSidecar) + blobSidecar == blobSidecar2 + not db.getBlobSidecarSZ(blockRoot0, 3, buf) + db.getBlobSidecarSZ(blockRoot0, 2, buf) + db.getBlobSidecarSZ(blockRoot1, 2, buf) + + check db.delBlobSidecar(blockRoot0, 2) + + check: + not db.getBlobSidecar(blockRoot0, 3, blobSidecar) + not db.getBlobSidecar(blockRoot0, 2, blobSidecar) + db.getBlobSidecar(blockRoot1, 2, blobSidecar) + blobSidecar == blobSidecar2 + not db.getBlobSidecarSZ(blockRoot0, 3, buf) + not db.getBlobSidecarSZ(blockRoot0, 2, buf) + db.getBlobSidecarSZ(blockRoot1, 2, buf) + + check db.delBlobSidecar(blockRoot1, 2) + + check: + not db.getBlobSidecar(blockRoot0, 3, blobSidecar) + not db.getBlobSidecar(blockRoot0, 2, blobSidecar) + not db.getBlobSidecar(blockRoot1, 2, blobSidecar) + not db.getBlobSidecarSZ(blockRoot0, 3, buf) + not db.getBlobSidecarSZ(blockRoot0, 2, buf) + not db.getBlobSidecarSZ(blockRoot1, 2, buf) + + db.close() + suite "FinalizedBlocks" & preset(): test "Basic ops" & preset(): var diff --git a/tests/testdbutil.nim b/tests/testdbutil.nim index 1545deb895..20cfe623b4 100644 --- a/tests/testdbutil.nim +++ b/tests/testdbutil.nim @@ -23,6 +23,11 @@ proc makeTestDB*( eth1Data = Opt.none(Eth1Data), flags: UpdateFlags = {}, cfg = defaultRuntimeConfig): BeaconChainDB = + # Blob support requires DENEB_FORK_EPOCH != FAR_FUTURE_EPOCH + var cfg = cfg + cfg.CAPELLA_FORK_EPOCH = 90000.Epoch + cfg.DENEB_FORK_EPOCH = 100000.Epoch + var genState = (ref ForkedHashedBeaconState)( kind: ConsensusFork.Phase0, phase0Data: initialize_hashed_beacon_state_from_eth1( From 922283e2e39e21d507f2197d1db7d18afdfc986e Mon Sep 17 00:00:00 2001 From: tersec Date: Thu, 26 Oct 2023 23:16:15 +0000 Subject: [PATCH 02/35] clean up ncli_testnet and reduce `ValidIpAddress` usage (#5529) * ValidIpAddress -> IpAddress * clean up ncli_testnet and reduce ValidIpAddress usage --- beacon_chain/nimbus_binary_common.nim | 2 +- ncli/ncli_testnet.nim | 36 ++++++++++++++++----------- tests/test_discovery.nim | 13 +++++++--- vendor/nim-confutils | 2 +- 4 files changed, 33 insertions(+), 20 deletions(-) diff --git a/beacon_chain/nimbus_binary_common.nim b/beacon_chain/nimbus_binary_common.nim index 82995e99f3..73ada81de1 100644 --- a/beacon_chain/nimbus_binary_common.nim +++ b/beacon_chain/nimbus_binary_common.nim @@ -351,7 +351,7 @@ proc runSlotLoop*[T](node: T, startTime: BeaconTime, timeToNextSlot = nextSlot.start_beacon_time() - node.beaconClock.now() proc init*(T: type RestServerRef, - ip: ValidIpAddress, + ip: IpAddress, port: Port, allowedOrigin: Option[string], validateFn: PatternCallback, diff --git a/ncli/ncli_testnet.nim b/ncli/ncli_testnet.nim index 4cb0c25a6a..0f0c6f8be8 100644 --- a/ncli/ncli_testnet.nim +++ b/ncli/ncli_testnet.nim @@ -8,7 +8,7 @@ {.push raises: [].} import - std/[os, sequtils, strutils, options, json, terminal, times], + std/[json, options], chronos, bearssl/rand, chronicles, confutils, stint, json_serialization, web3, web3/confutils_defs, eth/keys, eth/p2p/discoveryv5/random2, stew/[io2, byteutils], json_rpc/jsonmarshal, @@ -22,10 +22,18 @@ import ../tests/mocking/mock_genesis, ./logtrace +from std/os import changeFileExt, fileExists +from std/sequtils import mapIt, toSeq +from std/terminal import readPasswordFromStdin +from std/times import toUnix + # Compiled version of /scripts/depositContract.v.py in this repo # The contract was compiled in Remix (https://remix.ethereum.org/) with vyper (remote) compiler. const depositContractCode = staticRead "../beacon_chain/el/deposit_contract_code.txt" +# For nim-confutils, which uses this kind of init(Type, value) pattern +func init(T: type IpAddress, ip: IpAddress): T = ip + type Eth1Address = web3.Address @@ -338,7 +346,7 @@ func `as`(blk: BlockObject, T: type deneb.ExecutionPayloadHeader): T = blob_gas_used: uint64 blk.blobGasUsed.getOrDefault(), excess_blob_gas: uint64 blk.excessBlobGas.getOrDefault()) -proc createDepositTreeSnapshot(deposits: seq[DepositData], +func createDepositTreeSnapshot(deposits: seq[DepositData], blockHash: Eth2Digest, blockHeight: uint64): DepositTreeSnapshot = var merkleizer = DepositsMerkleizer.init() @@ -357,7 +365,7 @@ proc createEnr(rng: var HmacDrbgContext, netKeyInsecurePassword: bool, cfg: RuntimeConfig, forkId: seq[byte], - address: ValidIpAddress, + address: IpAddress, port: Port): enr.Record {.raises: [CatchableError].} = type MetaData = altair.MetaData @@ -369,7 +377,7 @@ proc createEnr(rng: var HmacDrbgContext, bootstrapEnr = enr.Record.init( 1, # sequence number networkKeys.seckey.asEthKey, - some(address), + some(ValidIpAddress.init address), some(port), some(port), [ @@ -378,9 +386,9 @@ proc createEnr(rng: var HmacDrbgContext, ]) bootstrapEnr.tryGet() -proc doCreateTestnetEnr*(config: CliConfig, - rng: var HmacDrbgContext) - {.raises: [CatchableError].} = +proc doCreateTestnetEnr(config: CliConfig, + rng: var HmacDrbgContext) + {.raises: [CatchableError].} = let cfg = getRuntimeConfig(config.eth2Network) bootstrapEnr = parseBootstrapAddress(toSeq(lines(string config.inputBootstrapEnr))[0]).get() @@ -500,7 +508,7 @@ proc doCreateTestnet*(config: CliConfig, writeFile(bootstrapFile, enr.toURI) echo "Wrote ", bootstrapFile -proc deployContract*(web3: Web3, code: string): Future[ReceiptObject] {.async.} = +proc deployContract(web3: Web3, code: string): Future[ReceiptObject] {.async.} = var code = code if code[1] notin {'x', 'X'}: code = "0x" & code @@ -523,9 +531,9 @@ proc sendEth(web3: Web3, to: Eth1Address, valueEth: int): Future[TxHash] = web3.send(tr) type - DelayGenerator* = proc(): chronos.Duration {.gcsafe, raises: [].} + DelayGenerator = proc(): chronos.Duration {.gcsafe, raises: [].} -proc ethToWei(eth: UInt256): UInt256 = +func ethToWei(eth: UInt256): UInt256 = eth * 1000000000000000000.u256 proc initWeb3(web3Url, privateKey: string): Future[Web3] {.async.} = @@ -539,10 +547,10 @@ proc initWeb3(web3Url, privateKey: string): Future[Web3] {.async.} = # TODO: async functions should note take `seq` inputs because # this leads to full copies. -proc sendDeposits*(deposits: seq[LaunchPadDeposit], - web3Url, privateKey: string, - depositContractAddress: Eth1Address, - delayGenerator: DelayGenerator = nil) {.async.} = +proc sendDeposits(deposits: seq[LaunchPadDeposit], + web3Url, privateKey: string, + depositContractAddress: Eth1Address, + delayGenerator: DelayGenerator = nil) {.async.} = notice "Sending deposits", web3 = web3Url, depositContract = depositContractAddress diff --git a/tests/test_discovery.nim b/tests/test_discovery.nim index 7888f7c581..2c43bc369c 100644 --- a/tests/test_discovery.nim +++ b/tests/test_discovery.nim @@ -16,18 +16,23 @@ import proc new(T: type Eth2DiscoveryProtocol, pk: keys.PrivateKey, - enrIp: Option[ValidIpAddress], enrTcpPort, enrUdpPort: Option[Port], - bindPort: Port, bindIp: ValidIpAddress, + enrIp: Option[IpAddress], enrTcpPort, enrUdpPort: Option[Port], + bindPort: Port, bindIp: IpAddress, enrFields: openArray[(string, seq[byte])] = [], rng: ref HmacDrbgContext): T {.raises: [CatchableError].} = + let optValidIpAddress = + if enrIp.isSome: + some ValidIpAddress.init enrIp.get + else: + none ValidIpAddress - newProtocol(pk, enrIp, enrTcpPort, enrUdpPort, enrFields, + newProtocol(pk, optValidIpAddress, enrTcpPort, enrUdpPort, enrFields, bindPort = bindPort, bindIp = bindIp, rng = rng) proc generateNode(rng: ref HmacDrbgContext, port: Port, enrFields: openArray[(string, seq[byte])] = []): Eth2DiscoveryProtocol = - let ip = ValidIpAddress.init("127.0.0.1") + let ip = parseIpAddress("127.0.0.1") Eth2DiscoveryProtocol.new(keys.PrivateKey.random(rng[]), some(ip), some(port), some(port), port, ip, enrFields, rng = rng) diff --git a/vendor/nim-confutils b/vendor/nim-confutils index 674c9e4c8e..07b598ff28 160000 --- a/vendor/nim-confutils +++ b/vendor/nim-confutils @@ -1 +1 @@ -Subproject commit 674c9e4c8e0cad2b7193cc9a59c12d39a397750f +Subproject commit 07b598ff2875f5c86da6546f075d3ed7a1a45b0f From a0e00637556b133e60c1cd0cf165e101e0c3dc81 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Fri, 27 Oct 2023 19:05:18 +0200 Subject: [PATCH 03/35] bump `nim-serialization` to `543b2f3dd0724f7cf631feba6c2a3ec438f3d230` (#5535) - workaround `--mm:orc` codegen bug with `{.noSideEffect.}` --- vendor/nim-serialization | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/nim-serialization b/vendor/nim-serialization index 4bdbc29e54..543b2f3dd0 160000 --- a/vendor/nim-serialization +++ b/vendor/nim-serialization @@ -1 +1 @@ -Subproject commit 4bdbc29e54fe54049950e352bb969aab97173b35 +Subproject commit 543b2f3dd0724f7cf631feba6c2a3ec438f3d230 From 48d7e62335e191ff2ac74c68847c676d7f77cda1 Mon Sep 17 00:00:00 2001 From: tersec Date: Fri, 27 Oct 2023 21:54:48 +0000 Subject: [PATCH 04/35] bump nim-confutils to facilitate IpAddress usage (#5533) --- vendor/nim-confutils | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/nim-confutils b/vendor/nim-confutils index 07b598ff28..7568f1b7c3 160000 --- a/vendor/nim-confutils +++ b/vendor/nim-confutils @@ -1 +1 @@ -Subproject commit 07b598ff2875f5c86da6546f075d3ed7a1a45b0f +Subproject commit 7568f1b7c3142d8e87c1f3dd42924238926affbe From 34eade3a992634d45c53a211a22274fae760794b Mon Sep 17 00:00:00 2001 From: tersec Date: Sat, 28 Oct 2023 05:10:23 +0000 Subject: [PATCH 05/35] sign blinded blob sidecars during builder API proposals (#5537) --- beacon_chain/spec/signatures.nim | 5 +++-- beacon_chain/validators/beacon_validators.nim | 11 ++++++++++- beacon_chain/validators/validator_pool.nim | 3 ++- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/beacon_chain/spec/signatures.nim b/beacon_chain/spec/signatures.nim index 5b756d64e8..1e456db629 100644 --- a/beacon_chain/spec/signatures.nim +++ b/beacon_chain/spec/signatures.nim @@ -90,7 +90,7 @@ func compute_block_signing_root*( func compute_blob_signing_root( fork: Fork, genesis_validators_root: Eth2Digest, slot: Slot, - blob: BlobSidecar): Eth2Digest = + blob: BlindedBlobSidecar | BlobSidecar): Eth2Digest = let epoch = epoch(slot) domain = get_domain(fork, DOMAIN_BLOB_SIDECAR, epoch, @@ -109,7 +109,8 @@ func get_block_signature*( # https://github.com/ethereum/consensus-specs/blob/v1.4.0-alpha.3/specs/deneb/validator.md#constructing-the-signedblobsidecars proc get_blob_sidecar_signature*( fork: Fork, genesis_validators_root: Eth2Digest, slot: Slot, - blob: BlobSidecar, privkey: ValidatorPrivKey): CookedSig = + blob: BlindedBlobSidecar | BlobSidecar, privkey: ValidatorPrivKey): + CookedSig = let signing_root = compute_blob_signing_root( fork, genesis_validators_root, slot, blob) diff --git a/beacon_chain/validators/beacon_validators.nim b/beacon_chain/validators/beacon_validators.nim index ee3b2da7d7..3eeba31ea4 100644 --- a/beacon_chain/validators/beacon_validators.nim +++ b/beacon_chain/validators/beacon_validators.nim @@ -761,9 +761,18 @@ proc blindedBlockCheckSlashingAndSign[ fork, genesis_validators_root, slot, blockRoot, blindedBlockContents.signed_blinded_block.message) if res.isErr(): - return err("Unable to sign block: " & res.error()) + return err("Unable to sign blinded block: " & res.error()) res.get() + for signedBlindedBlobSidecar in mitems( + blindedBlockContents.signed_blinded_blob_sidecars): + signedBlindedBlobSidecar.signature = validator.getBlobSignature( + fork, genesis_validators_root, slot, + signedBlindedBlobSidecar.message).valueOr: + warn "Unable to sign blinded blob", + reason = error() + return + return ok blindedBlockContents proc getUnsignedBlindedBeaconBlock[ diff --git a/beacon_chain/validators/validator_pool.nim b/beacon_chain/validators/validator_pool.nim index ec3d750605..5ed1b52d03 100644 --- a/beacon_chain/validators/validator_pool.nim +++ b/beacon_chain/validators/validator_pool.nim @@ -628,7 +628,8 @@ proc getBlockSignature*(v: AttachedValidator, fork: Fork, # https://github.com/ethereum/consensus-specs/blob/v1.4.0-alpha.3/specs/deneb/validator.md#constructing-the-signedblobsidecars proc getBlobSignature*(v: AttachedValidator, fork: Fork, genesis_validators_root: Eth2Digest, slot: Slot, - blob: BlobSidecar): SignatureResult = + blob: BlindedBlobSidecar | BlobSidecar): + SignatureResult = return case v.kind of ValidatorKind.Local: From b0d50015710e6570d765a86bb2bff6d61b0a211d Mon Sep 17 00:00:00 2001 From: tersec Date: Sun, 29 Oct 2023 00:06:13 +0000 Subject: [PATCH 06/35] implement getBlobSidecars Beacon API endpoint (#5530) --- beacon_chain/rpc/rest_beacon_api.nim | 47 ++++++++++++++++++++++++++++ beacon_chain/rpc/rest_constants.nim | 3 +- ncli/resttest-rules.json | 24 ++++++++++++++ 3 files changed, 73 insertions(+), 1 deletion(-) diff --git a/beacon_chain/rpc/rest_beacon_api.nim b/beacon_chain/rpc/rest_beacon_api.nim index 876b48d4e4..352a46da61 100644 --- a/beacon_chain/rpc/rest_beacon_api.nim +++ b/beacon_chain/rpc/rest_beacon_api.nim @@ -1374,3 +1374,50 @@ proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) = VoluntaryExitValidationError, $res.error()) return RestApiResponse.jsonMsgResponse(VoluntaryExitValidationSuccess) + + # https://ethereum.github.io/beacon-APIs/?urls.primaryName=v2.4.2#/Beacon/getBlobSidecars + # https://github.com/ethereum/beacon-APIs/blob/v2.4.2/apis/beacon/blob_sidecars/blob_sidecars.yaml + router.api(MethodGet, "/eth/v1/beacon/blob_sidecars/{block_id}") do ( + block_id: BlockIdent, indices: seq[uint64]) -> RestApiResponse: + let + bid = block_id.valueOr: + return RestApiResponse.jsonError(Http400, InvalidBlockIdValueError, + $error) + + bdata = node.getForkedBlock(bid).valueOr: + return RestApiResponse.jsonError(Http404, BlockNotFoundError) + + contentType = block: + let res = preferredContentType(jsonMediaType, + sszMediaType) + if res.isErr(): + return RestApiResponse.jsonError(Http406, ContentNotAcceptableError) + res.get() + + # https://github.com/ethereum/beacon-APIs/blob/v2.4.2/types/deneb/blob_sidecar.yaml#L2-L28 + let data = newClone(default(List[BlobSidecar, Limit MAX_BLOBS_PER_BLOCK])) + + if indices.isErr: + return RestApiResponse.jsonError(Http400, + InvalidSidecarIndexValueError) + + let indexFilter = indices.get.toHashSet + + for blobIndex in 0'u64 ..< MAX_BLOBS_PER_BLOCK: + if indexFilter.len > 0 and blobIndex notin indexFilter: + continue + + var blobSidecar = new BlobSidecar + + if node.dag.db.getBlobSidecar(bdata.root, blobIndex, blobSidecar[]): + discard data[].add blobSidecar[] + + return + if contentType == sszMediaType: + RestApiResponse.sszResponse( + data[], headers = [("eth-consensus-version", + node.dag.cfg.consensusForkAtEpoch(bid.slot.epoch).toString())]) + elif contentType == jsonMediaType: + RestApiResponse.jsonResponse(data) + else: + RestApiResponse.jsonError(Http500, InvalidAcceptError) diff --git a/beacon_chain/rpc/rest_constants.nim b/beacon_chain/rpc/rest_constants.nim index 09e4bdfdf6..a57ddeb2ab 100644 --- a/beacon_chain/rpc/rest_constants.nim +++ b/beacon_chain/rpc/rest_constants.nim @@ -241,4 +241,5 @@ const "Failed to obtain fork information" InvalidTimestampValue* = "Invalid or missing timestamp value" - + InvalidSidecarIndexValueError* = + "Invalid blob index" diff --git a/ncli/resttest-rules.json b/ncli/resttest-rules.json index a0e593c853..8b09999086 100644 --- a/ncli/resttest-rules.json +++ b/ncli/resttest-rules.json @@ -2669,6 +2669,30 @@ "body": [{"operator": "jstructcmps", "start": ["data"],"value": [{"message": {"validator_index": "", "from_bls_pubkey": "", "to_execution_address": ""}, "signature": ""}]}] } }, + { + "topics": ["beacon", "blob_sidecars_blockid"], + "request": { + "url": "/eth/v1/beacon/blob_sidecars/head", + "headers": {"Accept": "application/json"} + }, + "response": {"status": {"operator": "equals", "value": "200"}} + }, + { + "topics": ["beacon", "blob_sidecars_blockid"], + "request": { + "url": "/eth/v1/beacon/blob_sidecars/finalized", + "headers": {"Accept": "application/json"} + }, + "response": {"status": {"operator": "equals", "value": "200"}} + }, + { + "topics": ["beacon", "blob_sidecars_blockid"], + "request": { + "url": "/eth/v1/beacon/blob_sidecars/0x0000000000000000000000000000000000000000000000000000000000000000", + "headers": {"Accept": "application/json"} + }, + "response": {"status": {"operator": "equals", "value": "404"}} + }, { "topics": ["builder", "states_expected_withdrawals"], "request": { From 62d59daaa7b7ad51b597fdd81000a4c9dc118e56 Mon Sep 17 00:00:00 2001 From: tersec Date: Mon, 30 Oct 2023 06:44:43 +0000 Subject: [PATCH 07/35] consensus-spec URL updates to v1.4.0-beta.3 (#5541) --- beacon_chain/beacon_chain_db_immutable.nim | 2 +- .../consensus_object_pools/attestation_pool.nim | 2 +- beacon_chain/consensus_object_pools/block_dag.nim | 2 +- .../consensus_object_pools/blockchain_dag.nim | 2 +- beacon_chain/consensus_object_pools/spec_cache.nim | 2 +- beacon_chain/el/el_manager.nim | 2 +- beacon_chain/gossip_processing/README.md | 2 +- .../gossip_processing/gossip_validation.nim | 4 ++-- beacon_chain/libnimbus_lc/libnimbus_lc.h | 14 +++++++------- beacon_chain/networking/eth2_network.nim | 4 ++-- beacon_chain/spec/beacon_time.nim | 2 +- beacon_chain/spec/beaconstate.nim | 6 +++--- beacon_chain/spec/datatypes/altair.nim | 4 ++-- beacon_chain/spec/datatypes/constants.nim | 2 +- beacon_chain/spec/datatypes/deneb.nim | 6 +++--- beacon_chain/spec/keystore.nim | 2 +- beacon_chain/spec/signatures.nim | 12 ++++++------ beacon_chain/spec/state_transition_block.nim | 2 +- beacon_chain/spec/state_transition_epoch.nim | 2 +- beacon_chain/validators/slashing_protection_v2.nim | 2 +- beacon_chain/validators/validator_pool.nim | 2 +- docs/attestation_flow.md | 2 +- tests/consensus_spec/fixtures_utils.nim | 2 +- 23 files changed, 41 insertions(+), 41 deletions(-) diff --git a/beacon_chain/beacon_chain_db_immutable.nim b/beacon_chain/beacon_chain_db_immutable.nim index cde1cd32d9..5c462715b2 100644 --- a/beacon_chain/beacon_chain_db_immutable.nim +++ b/beacon_chain/beacon_chain_db_immutable.nim @@ -127,7 +127,7 @@ type current_sync_committee*: SyncCommittee # [New in Altair] next_sync_committee*: SyncCommittee # [New in Altair] - # https://github.com/ethereum/consensus-specs/blob/v1.3.0/specs/bellatrix/beacon-chain.md#beaconstate + # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/bellatrix/beacon-chain.md#beaconstate # Memory-representation-equivalent to a Bellatrix BeaconState for in-place SSZ # reading and writing BellatrixBeaconStateNoImmutableValidators* = object diff --git a/beacon_chain/consensus_object_pools/attestation_pool.nim b/beacon_chain/consensus_object_pools/attestation_pool.nim index c5cb4c7cd6..5553823717 100644 --- a/beacon_chain/consensus_object_pools/attestation_pool.nim +++ b/beacon_chain/consensus_object_pools/attestation_pool.nim @@ -739,7 +739,7 @@ func getAggregatedAttestation*(pool: var AttestationPool, index: CommitteeIndex): Opt[Attestation] = ## Select the attestation that has the most votes going for it in the given ## slot/index - ## https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/phase0/validator.md#construct-aggregate + ## https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/phase0/validator.md#construct-aggregate let candidateIdx = pool.candidateIdx(slot) if candidateIdx.isNone: return Opt.none(Attestation) diff --git a/beacon_chain/consensus_object_pools/block_dag.nim b/beacon_chain/consensus_object_pools/block_dag.nim index b7e3756191..b18c7daf58 100644 --- a/beacon_chain/consensus_object_pools/block_dag.nim +++ b/beacon_chain/consensus_object_pools/block_dag.nim @@ -134,7 +134,7 @@ func link*(parent, child: BlockRef) = func get_ancestor*(blck: BlockRef, slot: Slot, maxDepth = 100'i64 * 365 * 24 * 60 * 60 div SECONDS_PER_SLOT.int): BlockRef = - ## https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/phase0/fork-choice.md#get_ancestor + ## https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/phase0/fork-choice.md#get_ancestor ## Return the most recent block as of the time at `slot` that not more recent ## than `blck` itself if isNil(blck): return nil diff --git a/beacon_chain/consensus_object_pools/blockchain_dag.nim b/beacon_chain/consensus_object_pools/blockchain_dag.nim index d601aa59f7..fc5cd5efdf 100644 --- a/beacon_chain/consensus_object_pools/blockchain_dag.nim +++ b/beacon_chain/consensus_object_pools/blockchain_dag.nim @@ -2613,7 +2613,7 @@ func aggregateAll*( # Aggregation spec requires non-empty collection # - https://tools.ietf.org/html/draft-irtf-cfrg-bls-signature-04 # Consensus specs require at least one attesting index in attestation - # - https://github.com/ethereum/consensus-specs/blob/v1.4.0-alpha.3/specs/phase0/beacon-chain.md#is_valid_indexed_attestation + # - https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/phase0/beacon-chain.md#is_valid_indexed_attestation return err("aggregate: no attesting keys") let diff --git a/beacon_chain/consensus_object_pools/spec_cache.nim b/beacon_chain/consensus_object_pools/spec_cache.nim index dd3b012573..f803e74c60 100644 --- a/beacon_chain/consensus_object_pools/spec_cache.nim +++ b/beacon_chain/consensus_object_pools/spec_cache.nim @@ -191,7 +191,7 @@ func makeAttestationData*( epoch: current_epoch, root: epoch_boundary_block.blck.root)) -# https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/phase0/validator.md#validator-assignments +# https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/phase0/validator.md#validator-assignments iterator get_committee_assignments*( shufflingRef: ShufflingRef, validator_indices: HashSet[ValidatorIndex]): tuple[committee_index: CommitteeIndex, diff --git a/beacon_chain/el/el_manager.nim b/beacon_chain/el/el_manager.nim index 928056057f..4c0d3517b2 100644 --- a/beacon_chain/el/el_manager.nim +++ b/beacon_chain/el/el_manager.nim @@ -423,7 +423,7 @@ template toGaugeValue(x: Quantity): int64 = # doAssert SECONDS_PER_ETH1_BLOCK * cfg.ETH1_FOLLOW_DISTANCE < GENESIS_DELAY, # "Invalid configuration: GENESIS_DELAY is set too low" -# https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/phase0/validator.md#get_eth1_data +# https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/phase0/validator.md#get_eth1_data func compute_time_at_slot(genesis_time: uint64, slot: Slot): uint64 = genesis_time + slot * SECONDS_PER_SLOT diff --git a/beacon_chain/gossip_processing/README.md b/beacon_chain/gossip_processing/README.md index 83f6eccba0..cd1cf4613b 100644 --- a/beacon_chain/gossip_processing/README.md +++ b/beacon_chain/gossip_processing/README.md @@ -9,7 +9,7 @@ This folder holds a collection of modules to: Gossip validation is different from consensus verification in particular for blocks. -- Blocks: https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/phase0/p2p-interface.md#beacon_block +- Blocks: https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/phase0/p2p-interface.md#beacon_block - Attestations (aggregated): https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/phase0/p2p-interface.md#beacon_aggregate_and_proof - Attestations (unaggregated): https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/phase0/p2p-interface.md#attestation-subnets - Voluntary exits: https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/phase0/p2p-interface.md#voluntary_exit diff --git a/beacon_chain/gossip_processing/gossip_validation.nim b/beacon_chain/gossip_processing/gossip_validation.nim index 939f4eec5a..3d58cf22eb 100644 --- a/beacon_chain/gossip_processing/gossip_validation.nim +++ b/beacon_chain/gossip_processing/gossip_validation.nim @@ -276,7 +276,7 @@ template validateBeaconBlockBellatrix( # # `is_merge_transition_complete(state)` tests for # `state.latest_execution_payload_header != ExecutionPayloadHeader()`, while - # https://github.com/ethereum/consensus-specs/blob/v1.3.0/specs/bellatrix/beacon-chain.md#block-processing + # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/bellatrix/beacon-chain.md#block-processing # shows that `state.latest_execution_payload_header` being default or not is # exactly equivalent to whether that block's execution payload is default or # not, so test cached block information rather than reconstructing a state. @@ -1012,7 +1012,7 @@ proc validateAttesterSlashing*( ok() -# https://github.com/ethereum/consensus-specs/blob/v1.3.0/specs/phase0/p2p-interface.md#proposer_slashing +# https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/phase0/p2p-interface.md#proposer_slashing proc validateProposerSlashing*( pool: ValidatorChangePool, proposer_slashing: ProposerSlashing): Result[void, ValidationError] = diff --git a/beacon_chain/libnimbus_lc/libnimbus_lc.h b/beacon_chain/libnimbus_lc/libnimbus_lc.h index f9bb665ab2..1c00d71bcc 100644 --- a/beacon_chain/libnimbus_lc/libnimbus_lc.h +++ b/beacon_chain/libnimbus_lc/libnimbus_lc.h @@ -149,11 +149,11 @@ typedef struct ETHBeaconState ETHBeaconState; * representation - If successful. * @return `NULL` - If the given `sszBytes` is malformed. * - * @see https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.2/specs/phase0/beacon-chain.md#beaconstate - * @see https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.2/specs/altair/beacon-chain.md#beaconstate - * @see https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.2/specs/bellatrix/beacon-chain.md#beaconstate - * @see https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.2/specs/capella/beacon-chain.md#beaconstate - * @see https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.2/configs/README.md + * @see https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/phase0/beacon-chain.md#beaconstate + * @see https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/altair/beacon-chain.md#beaconstate + * @see https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/bellatrix/beacon-chain.md#beaconstate + * @see https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/capella/beacon-chain.md#beaconstate + * @see https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/configs/README.md */ ETH_RESULT_USE_CHECK ETHBeaconState *ETHBeaconStateCreateFromSsz( @@ -595,7 +595,7 @@ const ETHLightClientHeader *ETHLightClientStoreGetFinalizedHeader( * @return Whether or not the next sync committee is currently known. * * @see https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/altair/light-client/sync-protocol.md#is_next_sync_committee_known - * @see https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/altair/light-client/light-client.md + * @see https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/altair/light-client/light-client.md */ ETH_RESULT_USE_CHECK bool ETHLightClientStoreIsNextSyncCommitteeKnown(const ETHLightClientStore *store); @@ -669,7 +669,7 @@ void ETHLightClientHeaderDestroy(ETHLightClientHeader *header); * * @return Pointer to a copy of the given header's beacon block root. * - * @see https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/phase0/beacon-chain.md#hash_tree_root + * @see https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/phase0/beacon-chain.md#hash_tree_root */ ETH_RESULT_USE_CHECK ETHRoot *ETHLightClientHeaderCopyBeaconRoot( diff --git a/beacon_chain/networking/eth2_network.nim b/beacon_chain/networking/eth2_network.nim index f7e2be3896..dd99abb3f9 100644 --- a/beacon_chain/networking/eth2_network.nim +++ b/beacon_chain/networking/eth2_network.nim @@ -174,7 +174,7 @@ type MounterProc* = proc(network: Eth2Node) {.gcsafe, raises: [CatchableError].} MessageContentPrinter* = proc(msg: pointer): string {.gcsafe, raises: [].} - # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/phase0/p2p-interface.md#goodbye + # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/phase0/p2p-interface.md#goodbye DisconnectionReason* = enum # might see other values on the wire! ClientShutDown = 1 @@ -2263,7 +2263,7 @@ proc getPersistentNetKeys*( func gossipId( data: openArray[byte], phase0Prefix, topic: string): seq[byte] = # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/phase0/p2p-interface.md#topics-and-messages - # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/altair/p2p-interface.md#topics-and-messages + # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/altair/p2p-interface.md#topics-and-messages const MESSAGE_DOMAIN_VALID_SNAPPY = [0x01'u8, 0x00, 0x00, 0x00] let messageDigest = withEth2Hash: h.update(MESSAGE_DOMAIN_VALID_SNAPPY) diff --git a/beacon_chain/spec/beacon_time.nim b/beacon_chain/spec/beacon_time.nim index f9281c73bd..46fd491106 100644 --- a/beacon_chain/spec/beacon_time.nim +++ b/beacon_chain/spec/beacon_time.nim @@ -188,7 +188,7 @@ func epoch*(slot: Slot): Epoch = # aka compute_epoch_at_slot if slot == FAR_FUTURE_SLOT: FAR_FUTURE_EPOCH else: Epoch(slot div SLOTS_PER_EPOCH) -# https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/phase0/fork-choice.md#compute_slots_since_epoch_start +# https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/phase0/fork-choice.md#compute_slots_since_epoch_start func since_epoch_start*(slot: Slot): uint64 = # aka compute_slots_since_epoch_start ## How many slots since the beginning of the epoch (`[0..SLOTS_PER_EPOCH-1]`) (slot mod SLOTS_PER_EPOCH) diff --git a/beacon_chain/spec/beaconstate.nim b/beacon_chain/spec/beaconstate.nim index 5c1499a366..19a01c58c2 100644 --- a/beacon_chain/spec/beaconstate.nim +++ b/beacon_chain/spec/beaconstate.nim @@ -142,7 +142,7 @@ func initiate_validator_exit*( from ./datatypes/deneb import BeaconState -# https://github.com/ethereum/consensus-specs/blob/v1.4.0-alpha.3/specs/phase0/beacon-chain.md#slash_validator +# https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/phase0/beacon-chain.md#slash_validator # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/altair/beacon-chain.md#modified-slash_validator # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/bellatrix/beacon-chain.md#modified-slash_validator func get_slashing_penalty*(state: ForkyBeaconState, @@ -342,7 +342,7 @@ func is_eligible_for_activation_queue*(validator: Validator): bool = validator.activation_eligibility_epoch == FAR_FUTURE_EPOCH and validator.effective_balance == MAX_EFFECTIVE_BALANCE -# https://github.com/ethereum/consensus-specs/blob/v1.4.0-alpha.3/specs/phase0/beacon-chain.md#is_eligible_for_activation +# https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/phase0/beacon-chain.md#is_eligible_for_activation func is_eligible_for_activation*( state: ForkyBeaconState, validator: Validator): bool = ## Check if ``validator`` is eligible for activation. @@ -1342,7 +1342,7 @@ func upgrade_to_capella*(cfg: RuntimeConfig, pre: bellatrix.BeaconState): # historical_summaries initialized to correct default automatically ) -# https://github.com/ethereum/consensus-specs/blob/v1.4.0-alpha.1/specs/deneb/fork.md#upgrading-the-state +# https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/deneb/fork.md#upgrading-the-state func upgrade_to_deneb*(cfg: RuntimeConfig, pre: capella.BeaconState): ref deneb.BeaconState = let diff --git a/beacon_chain/spec/datatypes/altair.nim b/beacon_chain/spec/datatypes/altair.nim index 89e9de5a97..4dfafa3f1e 100644 --- a/beacon_chain/spec/datatypes/altair.nim +++ b/beacon_chain/spec/datatypes/altair.nim @@ -102,7 +102,7 @@ type pubkeys*: HashArray[Limit SYNC_COMMITTEE_SIZE, ValidatorPubKey] aggregate_pubkey*: ValidatorPubKey - # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/altair/validator.md#synccommitteemessage + # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/altair/validator.md#synccommitteemessage SyncCommitteeMessage* = object slot*: Slot ## Slot to which this contribution pertains @@ -456,7 +456,7 @@ type SyncnetBits* = BitArray[SYNC_COMMITTEE_SUBNET_COUNT] - # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/altair/p2p-interface.md#metadata + # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/altair/p2p-interface.md#metadata MetaData* = object seq_number*: uint64 attnets*: AttnetBits diff --git a/beacon_chain/spec/datatypes/constants.nim b/beacon_chain/spec/datatypes/constants.nim index 703198c660..70c111193e 100644 --- a/beacon_chain/spec/datatypes/constants.nim +++ b/beacon_chain/spec/datatypes/constants.nim @@ -16,7 +16,7 @@ type DomainType* = distinct array[4, byte] const - # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/phase0/p2p-interface.md#constants + # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/phase0/p2p-interface.md#constants NODE_ID_BITS* = 256 # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/phase0/p2p-interface.md#configuration diff --git a/beacon_chain/spec/datatypes/deneb.nim b/beacon_chain/spec/datatypes/deneb.nim index d25c71f08b..b6d85b6365 100644 --- a/beacon_chain/spec/datatypes/deneb.nim +++ b/beacon_chain/spec/datatypes/deneb.nim @@ -47,13 +47,13 @@ type # current spec doesn't ever SSZ-serialize it or hash_tree_root it VersionedHash* = array[32, byte] - # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/deneb/beacon-chain.md#custom-types + # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/deneb/beacon-chain.md#custom-types BlobIndex* = uint64 # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.2/specs/deneb/polynomial-commitments.md#custom-types Blob* = array[BYTES_PER_FIELD_ELEMENT * FIELD_ELEMENTS_PER_BLOB, byte] - # https://github.com/ethereum/consensus-specs/blob/v1.3.0/specs/deneb/p2p-interface.md#blobsidecar + # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/deneb/p2p-interface.md#blobsidecar BlobSidecar* = object block_root*: Eth2Digest index*: BlobIndex @@ -110,7 +110,7 @@ type proofs*:seq[KZGProof] blobs*: Blobs - # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/deneb/beacon-chain.md#executionpayloadheader + # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/deneb/beacon-chain.md#executionpayloadheader ExecutionPayloadHeader* = object # Execution block header fields parent_hash*: Eth2Digest diff --git a/beacon_chain/spec/keystore.nim b/beacon_chain/spec/keystore.nim index dbeb55f577..68ae49ee74 100644 --- a/beacon_chain/spec/keystore.nim +++ b/beacon_chain/spec/keystore.nim @@ -1385,7 +1385,7 @@ proc createWallet*(kdfKind: KdfKind, crypto: crypto, nextAccount: nextAccount.get(0)) -# https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/phase0/validator.md#bls_withdrawal_prefix +# https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/phase0/validator.md#bls_withdrawal_prefix func makeWithdrawalCredentials*(k: ValidatorPubKey): Eth2Digest = var bytes = eth2digest(k.toRaw()) bytes.data[0] = BLS_WITHDRAWAL_PREFIX.uint8 diff --git a/beacon_chain/spec/signatures.nim b/beacon_chain/spec/signatures.nim index 1e456db629..1aa318537e 100644 --- a/beacon_chain/spec/signatures.nim +++ b/beacon_chain/spec/signatures.nim @@ -44,7 +44,7 @@ func compute_slot_signing_root*( fork, DOMAIN_SELECTION_PROOF, epoch, genesis_validators_root) compute_signing_root(slot, domain) -# https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/phase0/validator.md#aggregation-selection +# https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/phase0/validator.md#aggregation-selection func get_slot_signature*( fork: Fork, genesis_validators_root: Eth2Digest, slot: Slot, privkey: ValidatorPrivKey): CookedSig = @@ -97,7 +97,7 @@ func compute_blob_signing_root( genesis_validators_root) compute_signing_root(blob, domain) -# https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/phase0/validator.md#signature +# https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/phase0/validator.md#signature func get_block_signature*( fork: Fork, genesis_validators_root: Eth2Digest, slot: Slot, root: Eth2Digest, privkey: ValidatorPrivKey): CookedSig = @@ -106,7 +106,7 @@ func get_block_signature*( blsSign(privkey, signing_root.data) -# https://github.com/ethereum/consensus-specs/blob/v1.4.0-alpha.3/specs/deneb/validator.md#constructing-the-signedblobsidecars +# https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/deneb/validator.md#constructing-the-signedblobsidecars proc get_blob_sidecar_signature*( fork: Fork, genesis_validators_root: Eth2Digest, slot: Slot, blob: BlindedBlobSidecar | BlobSidecar, privkey: ValidatorPrivKey): @@ -147,7 +147,7 @@ func compute_aggregate_and_proof_signing_root*( fork, DOMAIN_AGGREGATE_AND_PROOF, epoch, genesis_validators_root) compute_signing_root(aggregate_and_proof, domain) -# https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/phase0/validator.md#broadcast-aggregate +# https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/phase0/validator.md#broadcast-aggregate func get_aggregate_and_proof_signature*(fork: Fork, genesis_validators_root: Eth2Digest, aggregate_and_proof: AggregateAndProof, privkey: ValidatorPrivKey): CookedSig = @@ -175,7 +175,7 @@ func compute_attestation_signing_root*( fork, DOMAIN_BEACON_ATTESTER, epoch, genesis_validators_root) compute_signing_root(attestation_data, domain) -# https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/phase0/validator.md#aggregate-signature +# https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/phase0/validator.md#aggregate-signature func get_attestation_signature*( fork: Fork, genesis_validators_root: Eth2Digest, attestation_data: AttestationData, @@ -265,7 +265,7 @@ proc verify_voluntary_exit_signature*( blsVerify(pubkey, signing_root.data, signature) -# https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/altair/validator.md#prepare-sync-committee-message +# https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/altair/validator.md#prepare-sync-committee-message func compute_sync_committee_message_signing_root*( fork: Fork, genesis_validators_root: Eth2Digest, slot: Slot, beacon_block_root: Eth2Digest): Eth2Digest = diff --git a/beacon_chain/spec/state_transition_block.nim b/beacon_chain/spec/state_transition_block.nim index 102661665e..21b027e565 100644 --- a/beacon_chain/spec/state_transition_block.nim +++ b/beacon_chain/spec/state_transition_block.nim @@ -747,7 +747,7 @@ func tx_peek_blob_versioned_hashes(opaque_tx: Transaction): # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/deneb/beacon-chain.md#kzg_commitment_to_versioned_hash func kzg_commitment_to_versioned_hash*( kzg_commitment: KzgCommitment): VersionedHash = - # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/deneb/beacon-chain.md#blob + # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/deneb/beacon-chain.md#blob const VERSIONED_HASH_VERSION_KZG = 0x01'u8 var res: VersionedHash diff --git a/beacon_chain/spec/state_transition_epoch.nim b/beacon_chain/spec/state_transition_epoch.nim index 1af9140920..8a1316a214 100644 --- a/beacon_chain/spec/state_transition_epoch.nim +++ b/beacon_chain/spec/state_transition_epoch.nim @@ -1152,7 +1152,7 @@ proc process_epoch*( info.init(state) - # https://github.com/ethereum/consensus-specs/blob/v1.3.0/specs/altair/beacon-chain.md#justification-and-finalization + # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/altair/beacon-chain.md#justification-and-finalization # [Modified in Altair] process_justification_and_finalization(state, info.balances, flags) diff --git a/beacon_chain/validators/slashing_protection_v2.nim b/beacon_chain/validators/slashing_protection_v2.nim index c6202f88c9..4f39d5770a 100644 --- a/beacon_chain/validators/slashing_protection_v2.nim +++ b/beacon_chain/validators/slashing_protection_v2.nim @@ -33,7 +33,7 @@ import # - https://notes.ethereum.org/@djrtwo/Bkn3zpwxB#Validator-responsibilities # # Phase 0 spec - Honest Validator - how to avoid slashing -# - https://github.com/ethereum/consensus-specs/blob/v1.3.0/specs/phase0/validator.md#how-to-avoid-slashing +# - https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/phase0/validator.md#how-to-avoid-slashing # # In-depth reading on slashing conditions # diff --git a/beacon_chain/validators/validator_pool.nim b/beacon_chain/validators/validator_pool.nim index 5ed1b52d03..345712e475 100644 --- a/beacon_chain/validators/validator_pool.nim +++ b/beacon_chain/validators/validator_pool.nim @@ -640,7 +640,7 @@ proc getBlobSignature*(v: AttachedValidator, fork: Fork, of ValidatorKind.Remote: return SignatureResult.err("web3signer not supported for blobs") -# https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/phase0/validator.md#aggregate-signature +# https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/phase0/validator.md#aggregate-signature proc getAttestationSignature*(v: AttachedValidator, fork: Fork, genesis_validators_root: Eth2Digest, data: AttestationData diff --git a/docs/attestation_flow.md b/docs/attestation_flow.md index f0a483ba56..14ce970cab 100644 --- a/docs/attestation_flow.md +++ b/docs/attestation_flow.md @@ -52,7 +52,7 @@ These GossipSub topics are used to listen for attestations: The attestations are then validated by `validateAttestation()` or `validateAggregate()` in either `attestationValidator()` or `aggregateValidator()` according to the P2P specs. - https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/phase0/p2p-interface.md#beacon_aggregate_and_proof -- https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/phase0/p2p-interface.md#attestation-subnets +- https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/phase0/p2p-interface.md#attestation-subnets Finally, valid attestations are added to the local `attestationPool`. Attestations are dropped in case of an error. diff --git a/tests/consensus_spec/fixtures_utils.nim b/tests/consensus_spec/fixtures_utils.nim index e62c780f36..4682378721 100644 --- a/tests/consensus_spec/fixtures_utils.nim +++ b/tests/consensus_spec/fixtures_utils.nim @@ -75,7 +75,7 @@ type rewards*: List[uint64, Limit VALIDATOR_REGISTRY_LIMIT] penalties*: List[uint64, Limit VALIDATOR_REGISTRY_LIMIT] - # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/phase0/validator.md#eth1block + # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/phase0/validator.md#eth1block Eth1Block* = object timestamp*: uint64 deposit_root*: Eth2Digest From d289da8cd497d717b0608fd56fd0bcf73a11a54c Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Tue, 31 Oct 2023 01:56:52 +0100 Subject: [PATCH 08/35] extract trusted node sync trigger into separate function (#5544) To prepare for calling trusted node sync from the main Nimbus startup logic, extract the trusted node sync trigger into a separate function. Further allow passing a pre-opened database, as that will be needed to check whether trusted node sync needs to be called during regular start. --- beacon_chain/nimbus_beacon_node.nim | 96 ++++++++++++++++++----------- beacon_chain/trusted_node_sync.nim | 11 ++-- 2 files changed, 64 insertions(+), 43 deletions(-) diff --git a/beacon_chain/nimbus_beacon_node.nim b/beacon_chain/nimbus_beacon_node.nim index 8506bcb744..553da4b4cf 100644 --- a/beacon_chain/nimbus_beacon_node.nim +++ b/beacon_chain/nimbus_beacon_node.nim @@ -114,6 +114,57 @@ declareGauge next_action_wait, logScope: topics = "beacnde" +proc doRunTrustedNodeSync( + db: BeaconChainDB, + metadata: Eth2NetworkMetadata, + databaseDir: string, + eraDir: string, + restUrl: string, + stateId: Option[string], + trustedBlockRoot: Option[Eth2Digest], + backfill: bool, + reindex: bool, + downloadDepositSnapshot: bool) {.async.} = + let + cfg = metadata.cfg + syncTarget = + if stateId.isSome: + if trustedBlockRoot.isSome: + warn "Ignoring `trustedBlockRoot`, `stateId` is set", + stateId, trustedBlockRoot + TrustedNodeSyncTarget( + kind: TrustedNodeSyncKind.StateId, + stateId: stateId.get) + elif trustedBlockRoot.isSome: + TrustedNodeSyncTarget( + kind: TrustedNodeSyncKind.TrustedBlockRoot, + trustedBlockRoot: trustedBlockRoot.get) + else: + TrustedNodeSyncTarget( + kind: TrustedNodeSyncKind.StateId, + stateId: "finalized") + genesis = + if metadata.hasGenesis: + let genesisBytes = try: await metadata.fetchGenesisBytes() + except CatchableError as err: + error "Failed to obtain genesis state", + source = metadata.genesis.sourceDesc, + err = err.msg + quit 1 + newClone(readSszForkedHashedBeaconState(cfg, genesisBytes)) + else: nil + + await db.doTrustedNodeSync( + cfg, + databaseDir, + eraDir, + restUrl, + syncTarget, + backfill, + reindex, + downloadDepositSnapshot, + genesis) + func getVanityLogs(stdoutKind: StdoutLogKind): VanityLogs = case stdoutKind of StdoutLogKind.Auto: raiseAssert "inadmissable here" @@ -2143,51 +2194,24 @@ proc handleStartUpCmd(config: var BeaconNodeConf) {.raises: [CatchableError].} = of BNStartUpCmd.web3: doWeb3Cmd(config, rng[]) of BNStartUpCmd.slashingdb: doSlashingInterchange(config) of BNStartUpCmd.trustedNodeSync: - let - network = loadEth2Network(config) - cfg = network.cfg - syncTarget = - if config.stateId.isSome: - if config.lcTrustedBlockRoot.isSome: - warn "Ignoring `trustedBlockRoot`, `stateId` is set", - stateId = config.stateId, - trustedBlockRoot = config.lcTrustedBlockRoot - TrustedNodeSyncTarget( - kind: TrustedNodeSyncKind.StateId, - stateId: config.stateId.get) - elif config.lcTrustedBlockRoot.isSome: - TrustedNodeSyncTarget( - kind: TrustedNodeSyncKind.TrustedBlockRoot, - trustedBlockRoot: config.lcTrustedBlockRoot.get) - else: - TrustedNodeSyncTarget( - kind: TrustedNodeSyncKind.StateId, - stateId: "finalized") - genesis = - if network.hasGenesis: - let genesisBytes = try: waitFor network.fetchGenesisBytes() - except CatchableError as err: - error "Failed to obtain genesis state", - source = network.genesis.sourceDesc, - err = err.msg - quit 1 - newClone(readSszForkedHashedBeaconState(cfg, genesisBytes)) - else: nil - if config.blockId.isSome(): error "--blockId option has been removed - use --state-id instead!" quit 1 - waitFor doTrustedNodeSync( - cfg, + let + metadata = loadEth2Network(config) + db = BeaconChainDB.new(config.databaseDir, metadata.cfg, inMemory = false) + waitFor db.doRunTrustedNodeSync( + metadata, config.databaseDir, config.eraDir, config.trustedNodeUrl, - syncTarget, + config.stateId, + config.lcTrustedBlockRoot, config.backfillBlocks, config.reindex, - config.downloadDepositSnapshot, - genesis) + config.downloadDepositSnapshot) + db.close() {.pop.} # TODO moduletests exceptions diff --git a/beacon_chain/trusted_node_sync.nim b/beacon_chain/trusted_node_sync.nim index cdd147761f..8e4851688c 100644 --- a/beacon_chain/trusted_node_sync.nim +++ b/beacon_chain/trusted_node_sync.nim @@ -68,6 +68,7 @@ func shortLog*(v: TrustedNodeSyncTarget): auto = chronicles.formatIt(TrustedNodeSyncTarget): shortLog(it) proc doTrustedNodeSync*( + db: BeaconChainDB, cfg: RuntimeConfig, databaseDir: string, eraDir: string, @@ -89,11 +90,6 @@ proc doTrustedNodeSync*( error "Cannot connect to server", error = error quit 1 - let - db = BeaconChainDB.new(databaseDir, cfg, inMemory = false) - defer: - db.close() - # If possible, we'll store the genesis state in the database - this is not # strictly necessary but renders the resulting database compatible with # versions prior to 22.11 and makes reindexing possible @@ -551,7 +547,8 @@ when isMainModule: kind: TrustedNodeSyncKind.StateId, stateId: os.paramStr(5)) backfill = os.paramCount() > 5 and os.paramStr(6) == "true" - - waitFor doTrustedNodeSync( + db = BeaconChainDB.new(databaseDir, cfg, inMemory = false) + waitFor db.doTrustedNodeSync( getRuntimeConfig(some os.paramStr(1)), os.paramStr(2), os.paramStr(3), os.paramStr(4), syncTarget, backfill, false, true) + db.close() From 8c81515bf68a02515050b2ec3118f42d2a87616e Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Tue, 31 Oct 2023 11:15:38 +0100 Subject: [PATCH 09/35] log slashing protection database error on failure to load (#5542) * log slashing protection database error on failure to load ...and fix noreturn warning * fix the copied one too * oops * oops 2 --- .../validators/message_router_mev.nim | 38 +++++++++---------- .../validators/slashing_protection_v2.nim | 12 +++--- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/beacon_chain/validators/message_router_mev.nim b/beacon_chain/validators/message_router_mev.nim index 8364a2d39b..7f6e6a6d85 100644 --- a/beacon_chain/validators/message_router_mev.nim +++ b/beacon_chain/validators/message_router_mev.nim @@ -72,7 +72,7 @@ proc unblindAndRouteBlockMEV*( if hash_tree_root( blindedBlock.message.body.execution_payload_header) != hash_tree_root(unblindedPayload.data.data): - return err("unblinded payload doesn't match blinded payload header: " & + err("unblinded payload doesn't match blinded payload header: " & $blindedBlock.message.body.execution_payload_header) else: # Signature provided is consistent with unblinded execution payload, @@ -107,19 +107,17 @@ proc unblindAndRouteBlockMEV*( blockRoot = shortLog(signedBlock.root), blck = shortLog(signedBlock), signature = shortLog(signedBlock.signature) - return ok newBlockRef + ok newBlockRef else: - return err("submitBlindedBlock failed with HTTP error code" & + # https://github.com/ethereum/builder-specs/blob/v0.3.0/specs/bellatrix/validator.md#proposer-slashing + # This means if a validator publishes a signature for a + # `BlindedBeaconBlock` (via a dissemination of a + # `SignedBlindedBeaconBlock`) then the validator **MUST** not use the + # local build process as a fallback, even in the event of some failure + # with the external builder network. + err("submitBlindedBlock failed with HTTP error code" & $unblindedPayload.status & ": " & $shortLog(blindedBlock)) - # https://github.com/ethereum/builder-specs/blob/v0.3.0/specs/bellatrix/validator.md#proposer-slashing - # This means if a validator publishes a signature for a - # `BlindedBeaconBlock` (via a dissemination of a - # `SignedBlindedBeaconBlock`) then the validator **MUST** not use the - # local build process as a fallback, even in the event of some failure - # with the external builder network. - return err("unblindAndRouteBlockMEV error") - # TODO currently cannot be combined into one generic function proc unblindAndRouteBlockMEV*( node: BeaconNode, payloadBuilderRestClient: RestClientRef, @@ -150,7 +148,7 @@ proc unblindAndRouteBlockMEV*( if hash_tree_root( blindedBlock.message.body.execution_payload_header) != hash_tree_root(unblindedPayload.data.data.execution_payload): - return err("unblinded payload doesn't match blinded payload header: " & + err("unblinded payload doesn't match blinded payload header: " & $blindedBlock.message.body.execution_payload_header) else: # Signature provided is consistent with unblinded execution payload, @@ -189,14 +187,14 @@ proc unblindAndRouteBlockMEV*( discard $denebImplementationMissing & ": route unblinded blobs" - return ok newBlockRef + ok newBlockRef else: - return err("submitBlindedBlock failed with HTTP error code" & + # https://github.com/ethereum/builder-specs/blob/v0.3.0/specs/bellatrix/validator.md#proposer-slashing + # This means if a validator publishes a signature for a + # `BlindedBeaconBlock` (via a dissemination of a + # `SignedBlindedBeaconBlock`) then the validator **MUST** not use the + # local build process as a fallback, even in the event of some failure + # with the external builder network. + err("submitBlindedBlock failed with HTTP error code" & $unblindedPayload.status & ": " & $shortLog(blindedBlock)) - # https://github.com/ethereum/builder-specs/blob/v0.3.0/specs/bellatrix/validator.md#proposer-slashing - # This means if a validator publishes a signature for a - # `BlindedBeaconBlock` (via a dissemination of a - # `SignedBlindedBeaconBlock`) then the validator **MUST** not use the - # local build process as a fallback, even in the event of some failure - # with the external builder network. diff --git a/beacon_chain/validators/slashing_protection_v2.nim b/beacon_chain/validators/slashing_protection_v2.nim index 4f39d5770a..f38928f815 100644 --- a/beacon_chain/validators/slashing_protection_v2.nim +++ b/beacon_chain/validators/slashing_protection_v2.nim @@ -674,8 +674,9 @@ proc initCompatV1*( let alreadyExists = fileExists(databasePath / databaseName & ".sqlite3") - backend = SqStoreRef.init(databasePath, databaseName).valueOr: - fatal "Failed to open slashing protection database" + backendRes = SqStoreRef.init(databasePath, databaseName) + backend = backendRes.valueOr: # TODO https://github.com/nim-lang/Nim/issues/22605 + fatal "Failed to open slashing protection database", err = backendRes.error quit 1 result.db = T(backend: backend) @@ -716,9 +717,10 @@ proc init*(T: type SlashingProtectionDB_v2, let alreadyExists = fileExists(databasePath / databaseName & ".sqlite3") - backend = SqStoreRef.init(databasePath, databaseName, - keyspaces = []).valueOr: - fatal "Failed to open slashing protection database" + backendRes = SqStoreRef.init(databasePath, databaseName, + keyspaces = []) + backend = backendRes.valueOr: # TODO https://github.com/nim-lang/Nim/issues/22605 + fatal "Failed to open slashing protection database", err = backendRes.error quit 1 result = T(backend: backend) From 173582ad70ddb6c0371f9c791b0d86d4680c096f Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Tue, 31 Oct 2023 13:43:46 +0100 Subject: [PATCH 10/35] initialize KZG crypto before starting network (#5543) Move KZG trusted setup initialization before `BeaconNode.init` to avoid edge case where network message is received and processed before crypto library has been properly initialized. Followup from #4870. --- beacon_chain/nimbus_beacon_node.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/beacon_chain/nimbus_beacon_node.nim b/beacon_chain/nimbus_beacon_node.nim index 553da4b4cf..7a45fd7b20 100644 --- a/beacon_chain/nimbus_beacon_node.nim +++ b/beacon_chain/nimbus_beacon_node.nim @@ -2066,9 +2066,7 @@ proc doRunBeaconNode(config: var BeaconNodeConf, rng: ref HmacDrbgContext) {.rai bnStatus = BeaconNodeStatus.Stopping c_signal(ansi_c.SIGTERM, SIGTERMHandler) - let node = waitFor BeaconNode.init(rng, config, metadata) - - if node.dag.cfg.DENEB_FORK_EPOCH != FAR_FUTURE_EPOCH: + if metadata.cfg.DENEB_FORK_EPOCH != FAR_FUTURE_EPOCH: let res = if config.trustedSetupFile.isNone: conf.loadKzgTrustedSetup() @@ -2077,6 +2075,8 @@ proc doRunBeaconNode(config: var BeaconNodeConf, rng: ref HmacDrbgContext) {.rai if res.isErr(): raiseAssert res.error() + let node = waitFor BeaconNode.init(rng, config, metadata) + if bnStatus == BeaconNodeStatus.Stopping: return From 556d5e71142080d57a3cb2a3a0fed311043f73fd Mon Sep 17 00:00:00 2001 From: tersec Date: Wed, 1 Nov 2023 05:53:09 +0100 Subject: [PATCH 11/35] rm unused code (#5538) --- .../consensus_object_pools/blockchain_dag.nim | 4 - .../blockchain_dag_light_client.nim | 6 +- beacon_chain/el/el_conf.nim | 3 - beacon_chain/el/el_manager.nim | 48 +++-------- beacon_chain/era_db.nim | 2 +- beacon_chain/networking/eth2_network.nim | 11 --- beacon_chain/networking/network_metadata.nim | 2 +- beacon_chain/spec/beaconstate.nim | 14 ---- beacon_chain/spec/forks.nim | 16 ++-- .../validators/keystore_management.nim | 10 +-- ncli/logtrace.nim | 79 +++---------------- 11 files changed, 35 insertions(+), 160 deletions(-) diff --git a/beacon_chain/consensus_object_pools/blockchain_dag.nim b/beacon_chain/consensus_object_pools/blockchain_dag.nim index fc5cd5efdf..4d5c8c6bfe 100644 --- a/beacon_chain/consensus_object_pools/blockchain_dag.nim +++ b/beacon_chain/consensus_object_pools/blockchain_dag.nim @@ -553,8 +553,6 @@ func init*( dag.putShufflingRef(tmp) tmp - attester_dependent_root = withState(state): - forkyState.attester_dependent_root total_active_balance = withState(state): get_total_active_balance(forkyState.data, cache) epochRef = EpochRef( @@ -2421,7 +2419,6 @@ proc updateHead*( if not(isNil(dag.onHeadChanged)): let - currentEpoch = epoch(newHead.slot) depRoot = withState(dag.headState): forkyState.proposer_dependent_root prevDepRoot = withState(dag.headState): forkyState.attester_dependent_root @@ -2733,7 +2730,6 @@ proc rebuildIndex*(dag: ChainDAGRef) = if state_root.isZero: # If we can find an era file with this state, use it as an alternative # starting point - ignore failures for now - var bytes: seq[byte] if dag.era.getState( historicalRoots, historicalSummaries, slot, state[]).isOk(): state_root = getStateRoot(state[]) diff --git a/beacon_chain/consensus_object_pools/blockchain_dag_light_client.nim b/beacon_chain/consensus_object_pools/blockchain_dag_light_client.nim index 6e32dd2645..b605f3e4ec 100644 --- a/beacon_chain/consensus_object_pools/blockchain_dag_light_client.nim +++ b/beacon_chain/consensus_object_pools/blockchain_dag_light_client.nim @@ -723,11 +723,7 @@ proc initLightClientDataCache*(dag: ChainDAGRef) = blocks.add bid # Process blocks (reuses `dag.headState`, but restores it to the current head) - var - tmpState = assignClone(dag.headState) - tmpCache, cache: StateCache - oldCheckpoint: Checkpoint - cpIndex = 0 + var cache: StateCache for i in countdown(blocks.high, blocks.low): bid = blocks[i] if not dag.updateExistingState( diff --git a/beacon_chain/el/el_conf.nim b/beacon_chain/el/el_conf.nim index 5f96105ca3..1458d46132 100644 --- a/beacon_chain/el/el_conf.nim +++ b/beacon_chain/el/el_conf.nim @@ -71,9 +71,6 @@ func unknownRoleMsg(role: string): string = template raiseError(reader: var TomlReader, msg: string) = raiseTomlErr(reader.lex, msg) -template raiseError(reader: var JsonReader, msg: string) = - raiseTomlErr(reader.lex, msg) - proc readValue*(reader: var TomlReader, value: var EngineApiRoles) {.raises: [SerializationError, IOError].} = let roles = reader.readValue seq[string] diff --git a/beacon_chain/el/el_manager.nim b/beacon_chain/el/el_manager.nim index 4c0d3517b2..e9334594f7 100644 --- a/beacon_chain/el/el_manager.nim +++ b/beacon_chain/el/el_manager.nim @@ -96,7 +96,6 @@ const type Eth1BlockNumber* = uint64 Eth1BlockTimestamp* = uint64 - Eth1BlockHeader = engine_api.BlockHeader Eth1Block* = ref object hash*: Eth2Digest @@ -393,9 +392,6 @@ template trackedRequestWithTimeout[T](connection: ELConnection, template cfg(m: ELManager): auto = m.eth1Chain.cfg -template db(m: ELManager): BeaconChainDB = - m.eth1Chain.db - func hasJwtSecret*(m: ELManager): bool = for c in m.elConnections: if c.engineUrl.jwtSecret.isSome: @@ -409,12 +405,6 @@ func isSynced*(m: ELManager): bool = template eth1ChainBlocks*(m: ELManager): Deque[Eth1Block] = m.eth1Chain.blocks -template finalizedDepositsMerkleizer(m: ELManager): auto = - m.eth1Chain.finalizedDepositsMerkleizer - -template headMerkleizer(m: ELManager): auto = - m.eth1Chain.headMerkleizer - template toGaugeValue(x: Quantity): int64 = toGaugeValue(distinctBase x) @@ -885,15 +875,6 @@ template EngineApiResponseType*(T: type capella.ExecutionPayloadForSigning): typ template EngineApiResponseType*(T: type deneb.ExecutionPayloadForSigning): type = engine_api.GetPayloadV3Response -template payload(response: engine_api.ExecutionPayloadV1): engine_api.ExecutionPayloadV1 = - response - -template payload(response: engine_api.GetPayloadV2Response): engine_api.ExecutionPayloadV1OrV2 = - response.executionPayload - -template payload(response: engine_api.GetPayloadV3Response): engine_api.ExecutionPayloadV3 = - response.executionPayload - template toEngineWithdrawals*(withdrawals: seq[capella.Withdrawal]): seq[WithdrawalV1] = mapIt(withdrawals, toEngineWithdrawal(it)) @@ -1810,10 +1791,6 @@ func new*(T: type ELConnection, engineUrl: engineUrl, depositContractSyncStatus: DepositContractSyncStatus.unknown) -template getOrDefault[T, E](r: Result[T, E]): T = - type TT = T - get(r, default(TT)) - proc init*(T: type Eth1Chain, cfg: RuntimeConfig, db: BeaconChainDB, @@ -2017,12 +1994,6 @@ proc syncBlockRange(m: ELManager, blockNumber = lastBlock.number, depositsProcessed = lastBlock.depositCount -func init(T: type FullBlockId, blk: Eth1BlockHeader|BlockObject): T = - FullBlockId(number: Eth1BlockNumber blk.number, hash: blk.hash) - -func isNewLastBlock(m: ELManager, blk: Eth1BlockHeader|BlockObject): bool = - m.latestEth1Block.isNone or blk.number.uint64 > m.latestEth1BlockNumber - func hasConnection*(m: ELManager): bool = m.elConnections.len > 0 @@ -2121,7 +2092,6 @@ proc syncEth1Chain(m: ELManager, connection: ELConnection) {.async.} = debug "Starting Eth1 syncing", `from` = shortLog(m.eth1Chain.blocks[^1]) - var didPollOnce = false while true: debug "syncEth1Chain tick" @@ -2182,7 +2152,7 @@ proc startChainSyncingLoop(m: ELManager) {.async.} = continue await syncEth1Chain(m, syncedConnectionFut.read) - except CatchableError as err: + except CatchableError: await sleepAsync(10.seconds) # A more detailed error is already logged by trackEngineApiRequest @@ -2238,17 +2208,17 @@ proc testWeb3Provider*(web3Url: Uri, stdout.write "\n" res - let - chainId = request "Chain ID": - web3.provider.eth_chainId() + discard request "Chain ID": + web3.provider.eth_chainId() + discard request "Sync status": + web3.provider.eth_syncing() + + let latestBlock = request "Latest block": web3.provider.eth_getBlockByNumber(blockId("latest"), false) - syncStatus = request "Sync status": - web3.provider.eth_syncing() - ns = web3.contractSender(DepositContract, depositContractAddress) - depositRoot = request "Deposit root": - ns.get_deposit_root.call(blockNumber = latestBlock.number.uint64) + discard request "Deposit root": + ns.get_deposit_root.call(blockNumber = latestBlock.number.uint64) diff --git a/beacon_chain/era_db.nim b/beacon_chain/era_db.nim index bd7c88a892..d5ab39b036 100644 --- a/beacon_chain/era_db.nim +++ b/beacon_chain/era_db.nim @@ -316,7 +316,7 @@ proc getBlock*( readSszBytes(tmp, result.get(), updateRoot = root.isNone) if root.isSome(): result.get().root = root.get() - except CatchableError as exc: + except CatchableError: result.err() proc getStateSZ*( diff --git a/beacon_chain/networking/eth2_network.nim b/beacon_chain/networking/eth2_network.nim index dd99abb3f9..1fa57e3ff3 100644 --- a/beacon_chain/networking/eth2_network.nim +++ b/beacon_chain/networking/eth2_network.nim @@ -1887,9 +1887,6 @@ proc new(T: type Eth2Node, node -template publicKey(node: Eth2Node): keys.PublicKey = - node.discovery.privKey.toPublicKey - proc startListening*(node: Eth2Node) {.async.} = if node.discoveryEnabled: try: @@ -2173,14 +2170,6 @@ proc peerTrimmerHeartbeat(node: Eth2Node) {.async.} = func asEthKey*(key: PrivateKey): keys.PrivateKey = keys.PrivateKey(key.skkey) -proc initAddress(T: type MultiAddress, str: string): T = - let address = MultiAddress.init(str) - if IPFS.match(address) and matchPartial(multiaddress.TCP, address): - result = address - else: - raise newException(MultiAddressError, - "Invalid bootstrap node multi-address") - template tcpEndPoint(address, port): auto = MultiAddress.init(address, tcpProtocol, port) diff --git a/beacon_chain/networking/network_metadata.nim b/beacon_chain/networking/network_metadata.nim index a7eea86e37..c56a13d30b 100644 --- a/beacon_chain/networking/network_metadata.nim +++ b/beacon_chain/networking/network_metadata.nim @@ -474,7 +474,7 @@ when const_preset in ["mainnet", "gnosis"]: toOpenArray(metadata.genesis.bakedBytes, 0, sizeof(BeaconStateHeader) - 1), BeaconStateHeader) Opt.some header.genesis_validators_root - except SerializationError as err: + except SerializationError: raiseAssert "Invalid baken-in genesis state" else: Opt.none Eth2Digest diff --git a/beacon_chain/spec/beaconstate.nim b/beacon_chain/spec/beaconstate.nim index 19a01c58c2..088bdeda55 100644 --- a/beacon_chain/spec/beaconstate.nim +++ b/beacon_chain/spec/beaconstate.nim @@ -1102,20 +1102,6 @@ proc initialize_beacon_state_from_eth1*( # TODO https://github.com/nim-lang/Nim/issues/19094 # state -proc initialize_hashed_beacon_state_from_eth1( - cfg: RuntimeConfig, - eth1_block_hash: Eth2Digest, - eth1_timestamp: uint64, - deposits: openArray[DepositData], - execution_payload_header: ForkyExecutionPayloadHeader, - flags: UpdateFlags = {}): auto = - # TODO https://github.com/nim-lang/Nim/issues/19094 - result = initHashedBeaconState( - initialize_beacon_state_from_eth1( - cfg, eth1_block_hash, eth1_timestamp, deposits, - execution_payload_header, flags)) - result.root = hash_tree_root(result.data) - # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/altair/fork.md#upgrading-the-state func translate_participation( state: var altair.BeaconState, diff --git a/beacon_chain/spec/forks.nim b/beacon_chain/spec/forks.nim index 5cd8f8c420..1360cd2f0e 100644 --- a/beacon_chain/spec/forks.nim +++ b/beacon_chain/spec/forks.nim @@ -646,11 +646,11 @@ template forky( template withEpochInfo*(x: ForkedEpochInfo, body: untyped): untyped = case x.kind of EpochInfoFork.Phase0: - const infoFork {.inject.} = EpochInfoFork.Phase0 + const infoFork {.inject, used.} = EpochInfoFork.Phase0 template info: untyped {.inject.} = x.phase0Data body of EpochInfoFork.Altair: - const infoFork {.inject.} = EpochInfoFork.Altair + const infoFork {.inject, used.} = EpochInfoFork.Altair template info: untyped {.inject.} = x.altairData body @@ -797,11 +797,11 @@ template withBlck*( case x.kind of ConsensusFork.Phase0: const consensusFork {.inject, used.} = ConsensusFork.Phase0 - template forkyBlck: untyped {.inject.} = x.phase0Data + template forkyBlck: untyped {.inject, used.} = x.phase0Data body of ConsensusFork.Altair: const consensusFork {.inject, used.} = ConsensusFork.Altair - template forkyBlck: untyped {.inject.} = x.altairData + template forkyBlck: untyped {.inject, used.} = x.altairData body of ConsensusFork.Bellatrix: const consensusFork {.inject, used.} = ConsensusFork.Bellatrix @@ -809,11 +809,11 @@ template withBlck*( body of ConsensusFork.Capella: const consensusFork {.inject, used.} = ConsensusFork.Capella - template forkyBlck: untyped {.inject.} = x.capellaData + template forkyBlck: untyped {.inject, used.} = x.capellaData body of ConsensusFork.Deneb: const consensusFork {.inject, used.} = ConsensusFork.Deneb - template forkyBlck: untyped {.inject.} = x.denebData + template forkyBlck: untyped {.inject, used.} = x.denebData body func proposer_index*(x: ForkedBeaconBlock): uint64 = @@ -899,8 +899,8 @@ template withStateAndBlck*( body of ConsensusFork.Phase0: const consensusFork {.inject.} = ConsensusFork.Phase0 - template forkyState: untyped {.inject.} = s.phase0Data - template forkyBlck: untyped {.inject.} = b.phase0Data + template forkyState: untyped {.inject, used.} = s.phase0Data + template forkyBlck: untyped {.inject, used.} = b.phase0Data body func toBeaconBlockHeader*( diff --git a/beacon_chain/validators/keystore_management.nim b/beacon_chain/validators/keystore_management.nim index 241bba84ad..ced5f2367b 100644 --- a/beacon_chain/validators/keystore_management.nim +++ b/beacon_chain/validators/keystore_management.nim @@ -1021,10 +1021,7 @@ proc createLocalValidatorFiles*( encodedStorage: string ): Result[void, KeystoreGenerationError] {.raises: [].} = - var - success = false # becomes true when everything is created successfully - cleanupSecretsDir = true # becomes false if secretsDir already existed - cleanupValidatorsDir = true # becomes false if validatorsDir already existed + var success = false # becomes true when everything is created successfully # secretsDir: let secretsDirExisted: bool = dirExists(secretsDir) @@ -1068,10 +1065,7 @@ proc createLockedLocalValidatorFiles( encodedStorage: string ): Result[FileLockHandle, KeystoreGenerationError] {.raises: [].} = - var - success = false # becomes true when everything is created successfully - cleanupSecretsDir = true # becomes false if secretsDir already existed - cleanupValidatorsDir = true # becomes false if validatorsDir already existed + var success = false # becomes true when everything is created successfully # secretsDir: let secretsDirExisted: bool = dirExists(secretsDir) diff --git a/ncli/logtrace.nim b/ncli/logtrace.nim index fdcdcb5749..0650a7c437 100644 --- a/ncli/logtrace.nim +++ b/ncli/logtrace.nim @@ -10,19 +10,6 @@ import from stew/io2 import IoErrorCode -const - LogTraceName = "Beacon-Chain LogTrace Tool" - LogTraceMajor: int = 0 - LogTraceMinor: int = 0 - LogTracePatch: int = 4 - LogTraceVersion = $LogTraceMajor & "." & $LogTraceMinor & "." & - $LogTracePatch - LogTraceCopyright = "Copyright(C) 2021-2023" & - " Status Research & Development GmbH" - LogTraceHeader = LogTraceName & ", Version " & LogTraceVersion & - " [" & hostOS & ": " & hostCPU & "]\r\n" & - LogTraceCopyright & "\r\n" - type StartUpCommand* {.pure.} = enum pubsub, asl, asr, aggasr, scmsr, csr, lat, traceAll, localSimChecks @@ -314,9 +301,6 @@ proc print(r: FileReport) = template fatal(issuesGroup: IssuesGroup, msg: string) = issuesGroup.fatalIssues.add msg -template warning(issuesGroup: IssuesGroup, msg: string) = - issuesGroup.warnings.add msg - proc new(T: type IssuesGroup, name: string): T = T(name: name) @@ -466,56 +450,6 @@ proc readLogFileForASRMessages(file: string, srnode: var SRANode, finally: stream.close() -proc readLogFileForSCMSendMessages(file: string, - ignoreErrors = true, - dumpErrors = false): seq[SlotMessage] = - var res = newSeq[SlotMessage]() - var stream = newFileStream(file) - var line: string - var counter = 0 - try: - while not(stream.atEnd()): - line = stream.readLine() - inc(counter) - var m: LogMessage - try: - m = Json.decode(line, LogMessage, allowUnknownFields = true) - except SerializationError as exc: - if dumpErrors: - error "Serialization error while reading file, ignoring", file = file, - line_number = counter, errorMsg = exc.formatMsg(line) - else: - error "Serialization error while reading file, ignoring", file = file, - line_number = counter - if not(ignoreErrors): - raise exc - else: - continue - - if m.msg == "Sync committee message sent": - let scmm = Json.decode(line, SCMSentMessage, - allowUnknownFields = true) - let m = SlotMessage(kind: SMessageType.SCMSent, - scmsmsg: scmm) - res.add(m) - elif m.msg == "Slot start": - let sm = Json.decode(line, SlotStartMessage, - allowUnknownFields = true) - let m = SlotMessage(kind: SMessageType.SlotStart, - ssmsg: sm) - res.add(m) - - if counter mod 10_000 == 0: - info "Processing file", file = extractFilename(file), - lines_processed = counter, - lines_filtered = len(res) - result = res - - except CatchableError as exc: - warn "Error reading data from file", file = file, errorMsg = exc.msg - finally: - stream.close() - proc readLogFileForSCMSRMessages(file: string, srnode: var SRSCNode, ignoreErrors = true, dumpErrors = false) = var stream = newFileStream(file) @@ -1191,6 +1125,19 @@ proc run*(conf: LogTraceConf) = quit ord(issuesDetected) when isMainModule: + const + LogTraceName = "Beacon-Chain LogTrace Tool" + LogTraceMajor: int = 0 + LogTraceMinor: int = 0 + LogTracePatch: int = 4 + LogTraceVersion = $LogTraceMajor & "." & $LogTraceMinor & "." & + $LogTracePatch + LogTraceCopyright = "Copyright(C) 2021-2023" & + " Status Research & Development GmbH" + LogTraceHeader = LogTraceName & ", Version " & LogTraceVersion & + " [" & hostOS & ": " & hostCPU & "]\r\n" & + LogTraceCopyright & "\r\n" + echo LogTraceHeader var conf = LogTraceConf.load(version = LogTraceVersion) run(conf) From 70aaeee7040edd3298e32f4528d4c392ebbfdc77 Mon Sep 17 00:00:00 2001 From: Eugene Kabanov Date: Wed, 1 Nov 2023 09:27:43 +0200 Subject: [PATCH 12/35] Enable comprehensive reasons when REST server faults to create server. (#5546) Bump chronos & presto. --- beacon_chain/nimbus_binary_common.nim | 33 ++++++++++++--------------- vendor/nim-chronos | 2 +- vendor/nim-presto | 2 +- 3 files changed, 16 insertions(+), 21 deletions(-) diff --git a/beacon_chain/nimbus_binary_common.nim b/beacon_chain/nimbus_binary_common.nim index 73ada81de1..7defa0d5b9 100644 --- a/beacon_chain/nimbus_binary_common.nim +++ b/beacon_chain/nimbus_binary_common.nim @@ -356,9 +356,10 @@ proc init*(T: type RestServerRef, allowedOrigin: Option[string], validateFn: PatternCallback, config: AnyConf): T = - let address = initTAddress(ip, port) - let serverFlags = {HttpServerFlags.QueryCommaSeparatedArray, - HttpServerFlags.NotifyDisconnect} + let + address = initTAddress(ip, port) + serverFlags = {HttpServerFlags.QueryCommaSeparatedArray, + HttpServerFlags.NotifyDisconnect} # We increase default timeout to help validator clients who poll our server # at least once per slot (12.seconds). let @@ -370,26 +371,20 @@ proc init*(T: type RestServerRef, maxHeadersSize = config.restMaxRequestHeadersSize * 1024 maxRequestBodySize = config.restMaxRequestBodySize * 1024 - let res = try: - RestServerRef.new(RestRouter.init(validateFn, allowedOrigin), - address, serverFlags = serverFlags, - httpHeadersTimeout = headersTimeout, - maxHeadersSize = maxHeadersSize, - maxRequestBodySize = maxRequestBodySize) - except CatchableError as err: - notice "Rest server could not be started", address = $address, - reason = err.msg - return nil - + let res = RestServerRef.new(RestRouter.init(validateFn, allowedOrigin), + address, serverFlags = serverFlags, + httpHeadersTimeout = headersTimeout, + maxHeadersSize = maxHeadersSize, + maxRequestBodySize = maxRequestBodySize, + errorType = string) if res.isErr(): - notice "Rest server could not be started", address = $address, + notice "REST HTTP server could not be started", address = $address, reason = res.error() nil else: - notice "Starting REST HTTP server", - url = "http://" & $ip & ":" & $port & "/" - - res.get() + let server = res.get() + notice "Starting REST HTTP server", url = "http://" & $server.localAddress() + server type KeymanagerInitResult* = object diff --git a/vendor/nim-chronos b/vendor/nim-chronos index 253bc3cfc0..be2edab3ac 160000 --- a/vendor/nim-chronos +++ b/vendor/nim-chronos @@ -1 +1 @@ -Subproject commit 253bc3cfc079de35f9b96b9934ce702605400a51 +Subproject commit be2edab3ac101da03a70cbf52bc3f3d972b35d91 diff --git a/vendor/nim-presto b/vendor/nim-presto index 2ae448ff5b..5ca16485e4 160000 --- a/vendor/nim-presto +++ b/vendor/nim-presto @@ -1 +1 @@ -Subproject commit 2ae448ff5b0808c8f562c6f0a70bbd7a05407a37 +Subproject commit 5ca16485e4d74e531d50d289ebc0f869d9e6352b From 1a5bcb479eae2be19e58bf119202b6bc7dd0bb6c Mon Sep 17 00:00:00 2001 From: Eugene Kabanov Date: Wed, 1 Nov 2023 09:31:18 +0200 Subject: [PATCH 13/35] Fix `broadcast_validation` handling in `publishBlockV2` (fixes #5531) (#5547) * Address issue #5531. * Add more tests. * Add to resttest ability to check values. Fix tests. --- beacon_chain/rpc/rest_beacon_api.nim | 29 +++-- beacon_chain/rpc/rest_constants.nim | 2 + .../eth2_apis/eth2_rest_serialization.nim | 21 ++++ beacon_chain/spec/eth2_apis/rest_types.nim | 3 + ncli/resttest-rules.json | 104 ++++++++++++++++++ ncli/resttest.nim | 42 +++++-- 6 files changed, 183 insertions(+), 18 deletions(-) diff --git a/beacon_chain/rpc/rest_beacon_api.nim b/beacon_chain/rpc/rest_beacon_api.nim index 352a46da61..56b741999b 100644 --- a/beacon_chain/rpc/rest_beacon_api.nim +++ b/beacon_chain/rpc/rest_beacon_api.nim @@ -906,19 +906,32 @@ proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) = # https://ethereum.github.io/beacon-APIs/#/Beacon/publishBlockV2 router.api(MethodPost, "/eth/v2/beacon/blocks") do ( + broadcast_validation: Option[BroadcastValidationType], contentBody: Option[ContentBody]) -> RestApiResponse: let res = block: - if contentBody.isNone(): - return RestApiResponse.jsonError(Http400, EmptyRequestBodyError) - if request.headers.getString("broadcast_validation") != "gossip": - # TODO (henridf): support 'consensus' and 'consensus_and_equivocation' - # broadcast_validation - return RestApiResponse.jsonError( - Http500, "gossip broadcast_validation only supported") let - body = contentBody.get() version = request.headers.getString("eth-consensus-version") + validation = + block: + let res = + if broadcast_validation.isNone(): + BroadcastValidationType.Gossip + else: + broadcast_validation.get().valueOr: + return RestApiResponse.jsonError(Http400, + InvalidBroadcastValidationType) + # TODO (henridf): support 'consensus' and + # 'consensus_and_equivocation' broadcast_validation types. + if res != BroadcastValidationType.Gossip: + return RestApiResponse.jsonError(Http500, + "Only `gossip` broadcast_validation option supported") + res + body = + block: + if contentBody.isNone(): + return RestApiResponse.jsonError(Http400, EmptyRequestBodyError) + contentBody.get() var restBlock = decodeBodyJsonOrSsz(RestPublishedSignedBlockContents, body, version).valueOr: diff --git a/beacon_chain/rpc/rest_constants.nim b/beacon_chain/rpc/rest_constants.nim index a57ddeb2ab..49819d7233 100644 --- a/beacon_chain/rpc/rest_constants.nim +++ b/beacon_chain/rpc/rest_constants.nim @@ -243,3 +243,5 @@ const "Invalid or missing timestamp value" InvalidSidecarIndexValueError* = "Invalid blob index" + InvalidBroadcastValidationType* = + "Invalid broadcast_validation type value" diff --git a/beacon_chain/spec/eth2_apis/eth2_rest_serialization.nim b/beacon_chain/spec/eth2_apis/eth2_rest_serialization.nim index 5498610c4d..b7d5789a38 100644 --- a/beacon_chain/spec/eth2_apis/eth2_rest_serialization.nim +++ b/beacon_chain/spec/eth2_apis/eth2_rest_serialization.nim @@ -3606,6 +3606,15 @@ proc encodeString*(value: StateIdent): RestResult[string] = of StateIdentType.Justified: ok("justified") +proc encodeString*(value: BroadcastValidationType): RestResult[string] = + case value + of BroadcastValidationType.Gossip: + ok("gossip") + of BroadcastValidationType.Consensus: + ok("consensus") + of BroadcastValidationType.ConsensusAndEquivocation: + ok("consensus_and_equivocation") + proc encodeString*(value: BlockIdent): RestResult[string] = case value.kind of BlockQueryKind.Slot: @@ -3821,6 +3830,18 @@ proc decodeString*(t: typedesc[BlockIdent], let res = ? Base10.decode(uint64, value) ok(BlockIdent(kind: BlockQueryKind.Slot, slot: Slot(res))) +proc decodeString*(t: typedesc[BroadcastValidationType], + value: string): Result[BroadcastValidationType, cstring] = + case value + of "gossip": + ok(BroadcastValidationType.Gossip) + of "consensus": + ok(BroadcastValidationType.Consensus) + of "consensus_and_equivocation": + ok(BroadcastValidationType.ConsensusAndEquivocation) + else: + err("Incorrect broadcast validation type value") + proc decodeString*(t: typedesc[ValidatorIdent], value: string): Result[ValidatorIdent, cstring] = if len(value) > 2: diff --git a/beacon_chain/spec/eth2_apis/rest_types.nim b/beacon_chain/spec/eth2_apis/rest_types.nim index 41459ceafb..1015cf55a9 100644 --- a/beacon_chain/spec/eth2_apis/rest_types.nim +++ b/beacon_chain/spec/eth2_apis/rest_types.nim @@ -84,6 +84,9 @@ type StateIdentType* {.pure.} = enum Head, Genesis, Finalized, Justified + BroadcastValidationType* {.pure.} = enum + Gossip, Consensus, ConsensusAndEquivocation + StateIdent* = object case kind*: StateQueryKind of StateQueryKind.Slot: diff --git a/ncli/resttest-rules.json b/ncli/resttest-rules.json index 8b09999086..18bcfdcd17 100644 --- a/ncli/resttest-rules.json +++ b/ncli/resttest-rules.json @@ -3611,5 +3611,109 @@ }} ] } + }, + { + "topics": ["beacon", "beacon_blocks_publish_v2"], + "request": { + "method": "POST", + "headers": { + "Accept": "application/json", + "Content-Type": "application/json", + "Eth-Consensus-Version": "capella" + }, + "url": "/eth/v2/beacon/blocks", + }, + "response": { + "status": {"operator": "equals", "value": "400"}, + "headers": [{"key": "Content-Type", "value": "application/json", "operator": "equals"}], + "body": [{"operator": "jstructcmpns", "value": {"code": 400, "message": "Empty request's body"}}] + } + }, + { + "topics": ["beacon", "beacon_blocks_publish_v2"], + "request": { + "method": "POST", + "headers": { + "Accept": "application/json", + "Content-Type": "application/json", + "Eth-Consensus-Version": "capella" + }, + "url": "/eth/v2/beacon/blocks?broadcast_validation=gossip", + }, + "response": { + "status": {"operator": "equals", "value": "400"}, + "headers": [{"key": "Content-Type", "value": "application/json", "operator": "equals"}], + "body": [{"operator": "jstructcmpnsav", "value": {"code": 400, "message": "Empty request's body"}}] + } + }, + { + "topics": ["beacon", "beacon_blocks_publish_v2"], + "request": { + "method": "POST", + "headers": { + "Accept": "application/json", + "Content-Type": "application/json", + "Eth-Consensus-Version": "capella" + }, + "url": "/eth/v2/beacon/blocks?broadcast_validation=test", + }, + "response": { + "status": {"operator": "equals", "value": "400"}, + "headers": [{"key": "Content-Type", "value": "application/json", "operator": "equals"}], + "body": [{"operator": "jstructcmpnsav", "value": {"code": 400, "message": "Invalid broadcast_validation type value"}}] + } + }, + { + "topics": ["beacon", "beacon_blocks_publish_v2"], + "request": { + "method": "POST", + "headers": { + "Accept": "application/json", + "Content-Type": "application/json", + "Eth-Consensus-Version": "capella" + }, + "url": "/eth/v2/beacon/blocks?broadcast_validation=", + }, + "response": { + "status": {"operator": "equals", "value": "400"}, + "headers": [{"key": "Content-Type", "value": "application/json", "operator": "equals"}], + "body": [{"operator": "jstructcmpnsav", "value": {"code": 400, "message": "Invalid broadcast_validation type value"}}] + } + }, + { + "topics": ["beacon", "beacon_blocks_publish_v2"], + "request": { + "method": "POST", + "headers": { + "Accept": "application/json", + "Content-Type": "application/json", + "Eth-Consensus-Version": "capella" + }, + "url": "/eth/v2/beacon/blocks?broadcast_validation=consensus", + }, + "comment": "TODO: This should be replaced when `consensus` become supported", + "response": { + "status": {"operator": "equals", "value": "500"}, + "headers": [{"key": "Content-Type", "value": "application/json", "operator": "equals"}], + "body": [{"operator": "jstructcmpnsav", "value": {"code": 500, "message": "Only `gossip` broadcast_validation option supported"}}] + } + }, + { + "topics": ["beacon", "beacon_blocks_publish_v2"], + "request": { + "method": "POST", + "headers": { + "Accept": "application/json", + "Content-Type": "application/json", + "Eth-Consensus-Version": "capella" + }, + "url": "/eth/v2/beacon/blocks?broadcast_validation=consensus_and_equivocation", + }, + "comment": "TODO: This should be replaced when `consensus_and_equivocation` become supported", + "response": { + "status": {"operator": "equals", "value": "500"}, + "headers": [{"key": "Content-Type", "value": "application/json", "operator": "equals"}], + "body": [{"operator": "jstructcmpnsav", "value": {"code": 500, "message": "Only `gossip` broadcast_validation option supported"}}] + } } ] diff --git a/ncli/resttest.nim b/ncli/resttest.nim index c8a9c47afc..c1150d91a8 100644 --- a/ncli/resttest.nim +++ b/ncli/resttest.nim @@ -34,7 +34,8 @@ type Exists, NotExists, Equals, OneOf, Substr BodyOperatorKind {.pure.} = enum - Exists, JsonStructCmpS, JsonStructCmpNS + Exists, JsonStructCmpS, JsonStructCmpNS, + JsonStructCmpSAV, JsonStructCmpNSAV StatusExpect = object kind: StatusOperatorKind @@ -649,13 +650,18 @@ proc getResponseBodyExpect(rule: JsonNode): Result[BodyExpect, cstring] = BodyOperatorKind.JsonStructCmpS of "jstructcmpns": BodyOperatorKind.JsonStructCmpNS + of "jstructcmpsav": + BodyOperatorKind.JsonStructCmpSAV + of "jstructcmpnsav": + BodyOperatorKind.JsonStructCmpNSAV else: return err("`response.body` element has incorrect operator") case operator of BodyOperatorKind.Exists: res.add(BodyItemExpect(kind: operator)) - of BodyOperatorKind.JsonStructCmpS, BodyOperatorKind.JsonStructCmpNS: + of BodyOperatorKind.JsonStructCmpS, BodyOperatorKind.JsonStructCmpNS, + BodyOperatorKind.JsonStructCmpSAV, BodyOperatorKind.JsonStructCmpNSAV: let start = block: var default: seq[string] @@ -762,7 +768,7 @@ proc getPath(jobj: JsonNode, path: seq[string]): Result[JsonNode, cstring] = jnode = jitem ok(jnode) -proc structCmp(j1, j2: JsonNode, strict: bool): bool = +proc structCmp(j1, j2: JsonNode, strict: bool, checkvalue: bool): bool = if j1.kind != j2.kind: return false case j1.kind @@ -776,7 +782,7 @@ proc structCmp(j1, j2: JsonNode, strict: bool): bool = false else: for item in j1.elems: - if not(structCmp(item, j2.elems[0], strict)): + if not(structCmp(item, j2.elems[0], strict, checkvalue)): return false true of JObject: @@ -787,7 +793,7 @@ proc structCmp(j1, j2: JsonNode, strict: bool): bool = let j2node = j2.getOrDefault(key) if isNil(j2node): return false - if not(structCmp(value, j2node, strict)): + if not(structCmp(value, j2node, strict, checkvalue)): return false true else: @@ -795,10 +801,18 @@ proc structCmp(j1, j2: JsonNode, strict: bool): bool = let j1node = j1.getOrDefault(key) if isNil(j1node): return false - if not(structCmp(j1node, value, strict)): + if not(structCmp(j1node, value, strict, checkvalue)): return false true - else: + of JString: + if checkvalue: j1.str == j2.str else: true + of JInt: + if checkvalue: j1.num == j2.num else: true + of JFloat: + if checkvalue: j1.fnum == j2.fnum else: true + of JBool: + if checkvalue: j1.bval == j2.bval else: true + of JNull: true proc validateBody(body: openArray[byte], expect: BodyExpect): bool = @@ -810,7 +824,8 @@ proc validateBody(body: openArray[byte], expect: BodyExpect): bool = of BodyOperatorKind.Exists: if len(body) == 0: return false - of BodyOperatorKind.JsonStructCmpS, BodyOperatorKind.JsonStructCmpNS: + of BodyOperatorKind.JsonStructCmpS, BodyOperatorKind.JsonStructCmpNS, + BodyOperatorKind.JsonStructCmpSAV, BodyOperatorKind.JsonStructCmpNSAV: let jbody = block: let jres = jsonBody(body) @@ -822,11 +837,18 @@ proc validateBody(body: openArray[byte], expect: BodyExpect): bool = return false jpathres.get() let strict = - if item.kind == BodyOperatorKind.JsonStructCmpS: + if item.kind in {BodyOperatorKind.JsonStructCmpS, + BodyOperatorKind.JsonStructCmpSAV}: + true + else: + false + let checkvalue = + if item.kind in {BodyOperatorKind.JsonStructCmpSAV, + BodyOperatorKind.JsonStructCmpNSAV}: true else: false - if not(structCmp(jbody, item.value, strict)): + if not(structCmp(jbody, item.value, strict, checkvalue)): return false true From e1f47bb27a5b529886de31b8c1bddbfcec02f1ed Mon Sep 17 00:00:00 2001 From: Eugene Kabanov Date: Wed, 1 Nov 2023 09:32:41 +0200 Subject: [PATCH 14/35] Address #5539. (#5548) --- beacon_chain/rpc/rest_node_api.nim | 16 ++++++---------- .../spec/eth2_apis/eth2_rest_serialization.nim | 10 ++++++++++ beacon_chain/spec/eth2_apis/rest_types.nim | 2 ++ ncli/resttest-rules.json | 18 +++++++++--------- 4 files changed, 27 insertions(+), 19 deletions(-) diff --git a/beacon_chain/rpc/rest_node_api.nim b/beacon_chain/rpc/rest_node_api.nim index f547db90ff..a747d04e74 100644 --- a/beacon_chain/rpc/rest_node_api.nim +++ b/beacon_chain/rpc/rest_node_api.nim @@ -176,24 +176,19 @@ proc installNodeApiHandlers*(router: var RestRouter, node: BeaconNode) = if state.isErr(): return RestApiResponse.jsonError(Http400, InvalidPeerStateValueError, $state.error()) - let sres = validateState(state.get()) - if sres.isErr(): + validateState(state.get()).valueOr: return RestApiResponse.jsonError(Http400, InvalidPeerStateValueError, - $sres.error()) - sres.get() + $error) let directionMask = block: if direction.isErr(): return RestApiResponse.jsonError(Http400, InvalidPeerDirectionValueError, $direction.error()) - let dres = validateDirection(direction.get()) - if dres.isErr(): + validateDirection(direction.get()).valueOr: return RestApiResponse.jsonError(Http400, InvalidPeerDirectionValueError, - $dres.error()) - dres.get() - + $error) var res: seq[RestNodePeer] for peer in node.network.peers.values(): if (peer.connectionState in connectionMask) and @@ -209,7 +204,8 @@ proc installNodeApiHandlers*(router: var RestRouter, node: BeaconNode) = proto: node.network.switch.peerStore[ProtoVersionBook][peer.peerId] ) res.add(peer) - return RestApiResponse.jsonResponseWMeta(res, (count: uint64(len(res)))) + return RestApiResponse.jsonResponseWMeta(res, + (count: RestNumeric(len(res)))) # https://ethereum.github.io/beacon-APIs/#/Node/getPeerCount router.api(MethodGet, "/eth/v1/node/peer_count") do () -> RestApiResponse: diff --git a/beacon_chain/spec/eth2_apis/eth2_rest_serialization.nim b/beacon_chain/spec/eth2_apis/eth2_rest_serialization.nim index b7d5789a38..e3e3d64f19 100644 --- a/beacon_chain/spec/eth2_apis/eth2_rest_serialization.nim +++ b/beacon_chain/spec/eth2_apis/eth2_rest_serialization.nim @@ -642,6 +642,16 @@ proc readValue*(reader: var JsonReader[RestJson], value: var uint8) {. else: reader.raiseUnexpectedValue($res.error() & ": " & svalue) +## RestNumeric +proc writeValue*(w: var JsonWriter[RestJson], + value: RestNumeric) {.raises: [IOError].} = + writeValue(w, int(value)) + +proc readValue*(reader: var JsonReader[RestJson], + value: var RestNumeric) {. + raises: [IOError, SerializationError].} = + value = RestNumeric(reader.readValue(int)) + ## JustificationBits proc writeValue*( w: var JsonWriter[RestJson], value: JustificationBits diff --git a/beacon_chain/spec/eth2_apis/rest_types.nim b/beacon_chain/spec/eth2_apis/rest_types.nim index 1015cf55a9..c36d25a610 100644 --- a/beacon_chain/spec/eth2_apis/rest_types.nim +++ b/beacon_chain/spec/eth2_apis/rest_types.nim @@ -116,6 +116,8 @@ type PeerDirectKind* {.pure.} = enum Inbound, Outbound + RestNumeric* = distinct int + RestAttesterDuty* = object pubkey*: ValidatorPubKey validator_index*: ValidatorIndex diff --git a/ncli/resttest-rules.json b/ncli/resttest-rules.json index 18bcfdcd17..59c249fee1 100644 --- a/ncli/resttest-rules.json +++ b/ncli/resttest-rules.json @@ -2899,7 +2899,7 @@ "response": { "status": {"operator": "equals", "value": "200"}, "headers": [{"key": "Content-Type", "value": "application/json", "operator": "equals"}], - "body": [{"operator": "jstructcmpns", "value": {"data": [{"peer_id": "", "enr": "", "last_seen_p2p_address": "", "state": "", "direction": ""}], "meta": {"count": ""}}}] + "body": [{"operator": "jstructcmpns", "value": {"data": [{"peer_id": "", "enr": "", "last_seen_p2p_address": "", "state": "", "direction": ""}], "meta": {"count": 0}}}] } }, { @@ -2911,7 +2911,7 @@ "response": { "status": {"operator": "equals", "value": "200"}, "headers": [{"key": "Content-Type", "value": "application/json", "operator": "equals"}], - "body": [{"operator": "jstructcmpns", "value": {"data": [{"peer_id": "", "enr": "", "last_seen_p2p_address": "", "state": "", "direction": ""}], "meta": {"count": ""}}}] + "body": [{"operator": "jstructcmpns", "value": {"data": [{"peer_id": "", "enr": "", "last_seen_p2p_address": "", "state": "", "direction": ""}], "meta": {"count": 0}}}] } }, { @@ -2923,7 +2923,7 @@ "response": { "status": {"operator": "equals", "value": "200"}, "headers": [{"key": "Content-Type", "value": "application/json", "operator": "equals"}], - "body": [{"operator": "jstructcmpns", "value": {"data": [{"peer_id": "", "enr": "", "last_seen_p2p_address": "", "state": "", "direction": ""}], "meta": {"count": ""}}}] + "body": [{"operator": "jstructcmpns", "value": {"data": [{"peer_id": "", "enr": "", "last_seen_p2p_address": "", "state": "", "direction": ""}], "meta": {"count": 0}}}] } }, { @@ -2935,7 +2935,7 @@ "response": { "status": {"operator": "equals", "value": "200"}, "headers": [{"key": "Content-Type", "value": "application/json", "operator": "equals"}], - "body": [{"operator": "jstructcmpns", "value": {"data": [{"peer_id": "", "enr": "", "last_seen_p2p_address": "", "state": "", "direction": ""}], "meta": {"count": ""}}}] + "body": [{"operator": "jstructcmpns", "value": {"data": [{"peer_id": "", "enr": "", "last_seen_p2p_address": "", "state": "", "direction": ""}], "meta": {"count": 0}}}] } }, { @@ -2947,7 +2947,7 @@ "response": { "status": {"operator": "equals", "value": "200"}, "headers": [{"key": "Content-Type", "value": "application/json", "operator": "equals"}], - "body": [{"operator": "jstructcmpns", "value": {"data": [{"peer_id": "", "enr": "", "last_seen_p2p_address": "", "state": "", "direction": ""}], "meta": {"count": ""}}}] + "body": [{"operator": "jstructcmpns", "value": {"data": [{"peer_id": "", "enr": "", "last_seen_p2p_address": "", "state": "", "direction": ""}], "meta": {"count": 0}}}] } }, { @@ -2959,7 +2959,7 @@ "response": { "status": {"operator": "equals", "value": "200"}, "headers": [{"key": "Content-Type", "value": "application/json", "operator": "equals"}], - "body": [{"operator": "jstructcmpns", "value": {"data": [{"peer_id": "", "enr": "", "last_seen_p2p_address": "", "state": "", "direction": ""}], "meta": {"count": ""}}}] + "body": [{"operator": "jstructcmpns", "value": {"data": [{"peer_id": "", "enr": "", "last_seen_p2p_address": "", "state": "", "direction": ""}], "meta": {"count": 0}}}] } }, { @@ -2971,7 +2971,7 @@ "response": { "status": {"operator": "equals", "value": "200"}, "headers": [{"key": "Content-Type", "value": "application/json", "operator": "equals"}], - "body": [{"operator": "jstructcmpns", "value": {"data": [{"peer_id": "", "enr": "", "last_seen_p2p_address": "", "state": "", "direction": ""}], "meta": {"count": ""}}}] + "body": [{"operator": "jstructcmpns", "value": {"data": [{"peer_id": "", "enr": "", "last_seen_p2p_address": "", "state": "", "direction": ""}], "meta": {"count": 0}}}] } }, { @@ -2983,7 +2983,7 @@ "response": { "status": {"operator": "equals", "value": "200"}, "headers": [{"key": "Content-Type", "value": "application/json", "operator": "equals"}], - "body": [{"operator": "jstructcmpns", "value": {"data": [{"peer_id": "", "enr": "", "last_seen_p2p_address": "", "state": "", "direction": ""}], "meta": {"count": ""}}}] + "body": [{"operator": "jstructcmpns", "value": {"data": [{"peer_id": "", "enr": "", "last_seen_p2p_address": "", "state": "", "direction": ""}], "meta": {"count": 0}}}] } }, { @@ -2995,7 +2995,7 @@ "response": { "status": {"operator": "equals", "value": "200"}, "headers": [{"key": "Content-Type", "value": "application/json", "operator": "equals"}], - "body": [{"operator": "jstructcmpns", "value": {"data": [{"peer_id": "", "enr": "", "last_seen_p2p_address": "", "state": "", "direction": ""}], "meta": {"count": ""}}}] + "body": [{"operator": "jstructcmpns", "value": {"data": [{"peer_id": "", "enr": "", "last_seen_p2p_address": "", "state": "", "direction": ""}], "meta": {"count": 0}}}] } }, { From ed446b1d5a81a191f20b39cab70cba4e63a1119f Mon Sep 17 00:00:00 2001 From: tersec Date: Wed, 1 Nov 2023 08:33:00 +0100 Subject: [PATCH 15/35] replace ValidIpAddress with IpAddress in configurations (#5536) --- beacon_chain/conf.nim | 24 ++++++++++++------------ beacon_chain/conf_light_client.nim | 2 +- beacon_chain/networking/eth2_network.nim | 6 ++++-- beacon_chain/nimbus_beacon_node.nim | 2 +- ncli/ncli_testnet.nim | 8 ++++---- 5 files changed, 22 insertions(+), 20 deletions(-) diff --git a/beacon_chain/conf.nim b/beacon_chain/conf.nim index 10ba6d1ccf..0692c2e091 100644 --- a/beacon_chain/conf.nim +++ b/beacon_chain/conf.nim @@ -36,7 +36,7 @@ from consensus_object_pools/block_pools_types_light_client export uri, nat, enr, - defaultEth2TcpPort, enabledLogLevel, ValidIpAddress, + defaultEth2TcpPort, enabledLogLevel, defs, parseCmdArg, completeCmdArg, network_metadata, el_conf, network, BlockHashOrNumber, confTomlDefs, confTomlNet, confTomlUri, @@ -47,8 +47,8 @@ declareGauge network_name, "network name", ["name"] const # TODO: How should we select between IPv4 and IPv6 # Maybe there should be a config option for this. - defaultListenAddress* = (static ValidIpAddress.init("0.0.0.0")) - defaultAdminListenAddress* = (static ValidIpAddress.init("127.0.0.1")) + defaultListenAddress* = (static parseIpAddress("0.0.0.0")) + defaultAdminListenAddress* = (static parseIpAddress("127.0.0.1")) defaultSigningNodeRequestTimeout* = 60 defaultBeaconNode* = "http://127.0.0.1:" & $defaultEth2RestPort defaultBeaconNodeUri* = parseUri(defaultBeaconNode) @@ -292,7 +292,7 @@ type desc: "Listening address for the Ethereum LibP2P and Discovery v5 traffic" defaultValue: defaultListenAddress defaultValueDesc: $defaultListenAddressDesc - name: "listen-address" .}: ValidIpAddress + name: "listen-address" .}: IpAddress tcpPort* {. desc: "Listening TCP port for Ethereum LibP2P traffic" @@ -408,7 +408,7 @@ type desc: "Listening address of the metrics server" defaultValue: defaultAdminListenAddress defaultValueDesc: $defaultAdminListenAddressDesc - name: "metrics-address" .}: ValidIpAddress + name: "metrics-address" .}: IpAddress metricsPort* {. desc: "Listening HTTP port of the metrics server" @@ -449,7 +449,7 @@ type # Deprecated > 1.7.0 hidden desc: "Deprecated for removal" - name: "rpc-address" .}: Option[ValidIpAddress] + name: "rpc-address" .}: Option[IpAddress] restEnabled* {. desc: "Enable the REST server" @@ -466,7 +466,7 @@ type desc: "Listening address of the REST server" defaultValue: defaultAdminListenAddress defaultValueDesc: $defaultAdminListenAddressDesc - name: "rest-address" .}: ValidIpAddress + name: "rest-address" .}: IpAddress restAllowedOrigin* {. desc: "Limit the access to the REST API to a particular hostname " & @@ -520,7 +520,7 @@ type desc: "Listening port for the REST keymanager API" defaultValue: defaultAdminListenAddress defaultValueDesc: $defaultAdminListenAddressDesc - name: "keymanager-address" .}: ValidIpAddress + name: "keymanager-address" .}: IpAddress keymanagerAllowedOrigin* {. desc: "Limit the access to the Keymanager API to a particular hostname " & @@ -776,7 +776,7 @@ type of RecordCmd.create: ipExt* {. desc: "External IP address" - name: "ip" .}: ValidIpAddress + name: "ip" .}: IpAddress tcpPortExt* {. desc: "External TCP port" @@ -973,7 +973,7 @@ type desc: "Listening port for the REST keymanager API" defaultValue: defaultAdminListenAddress defaultValueDesc: $defaultAdminListenAddressDesc - name: "keymanager-address" .}: ValidIpAddress + name: "keymanager-address" .}: IpAddress keymanagerAllowedOrigin* {. desc: "Limit the access to the Keymanager API to a particular hostname " & @@ -993,7 +993,7 @@ type desc: "Listening address of the metrics server (BETA)" defaultValue: defaultAdminListenAddress defaultValueDesc: $defaultAdminListenAddressDesc - name: "metrics-address" .}: ValidIpAddress + name: "metrics-address" .}: IpAddress metricsPort* {. desc: "Listening HTTP port of the metrics server (BETA)" @@ -1091,7 +1091,7 @@ type desc: "Listening address of the REST HTTP server" defaultValue: defaultAdminListenAddress defaultValueDesc: $defaultAdminListenAddressDesc - name: "bind-address" .}: ValidIpAddress + name: "bind-address" .}: IpAddress tlsEnabled* {. desc: "Use secure TLS communication for REST server" diff --git a/beacon_chain/conf_light_client.nim b/beacon_chain/conf_light_client.nim index 0b64b520dd..6279617501 100644 --- a/beacon_chain/conf_light_client.nim +++ b/beacon_chain/conf_light_client.nim @@ -65,7 +65,7 @@ type LightClientConf* = object desc: "Listening address for the Ethereum LibP2P and Discovery v5 traffic" defaultValue: defaultListenAddress defaultValueDesc: $defaultListenAddressDesc - name: "listen-address" .}: ValidIpAddress + name: "listen-address" .}: IpAddress tcpPort* {. desc: "Listening TCP port for Ethereum LibP2P traffic" diff --git a/beacon_chain/networking/eth2_network.nim b/beacon_chain/networking/eth2_network.nim index 1fa57e3ff3..d572ba005d 100644 --- a/beacon_chain/networking/eth2_network.nim +++ b/beacon_chain/networking/eth2_network.nim @@ -2304,7 +2304,8 @@ proc createEth2Node*(rng: ref HmacDrbgContext, cfg, getBeaconTime().slotOrZero.epoch, genesis_validators_root) (extIp, extTcpPort, extUdpPort) = try: setupAddress( - config.nat, config.listenAddress, config.tcpPort, config.udpPort, clientId) + config.nat, ValidIpAddress.init config.listenAddress, config.tcpPort, + config.udpPort, clientId) except CatchableError as exc: raise exc except Exception as exc: raiseAssert exc.msg @@ -2326,7 +2327,8 @@ proc createEth2Node*(rng: ref HmacDrbgContext, info "Adding privileged direct peer", peerId, address res - hostAddress = tcpEndPoint(config.listenAddress, config.tcpPort) + hostAddress = tcpEndPoint( + ValidIpAddress.init config.listenAddress, config.tcpPort) announcedAddresses = if extIp.isNone() or extTcpPort.isNone(): @[] else: @[tcpEndPoint(extIp.get(), extTcpPort.get())] diff --git a/beacon_chain/nimbus_beacon_node.nim b/beacon_chain/nimbus_beacon_node.nim index 7a45fd7b20..cb49eb300d 100644 --- a/beacon_chain/nimbus_beacon_node.nim +++ b/beacon_chain/nimbus_beacon_node.nim @@ -2108,7 +2108,7 @@ proc doRecord(config: BeaconNodeConf, rng: var HmacDrbgContext) {. let record = enr.Record.init( config.seqNumber, netKeys.seckey.asEthKey, - some(config.ipExt), + some(ValidIpAddress.init config.ipExt), some(config.tcpPortExt), some(config.udpPortExt), fieldPairs).expect("Record within size limits") diff --git a/ncli/ncli_testnet.nim b/ncli/ncli_testnet.nim index 0f0c6f8be8..c247aefb14 100644 --- a/ncli/ncli_testnet.nim +++ b/ncli/ncli_testnet.nim @@ -117,9 +117,9 @@ type bootstrapAddress* {. desc: "The public IP address that will be advertised as a bootstrap node for the testnet" - defaultValue: init(ValidIpAddress, defaultAdminListenAddress) + defaultValue: defaultAdminListenAddress defaultValueDesc: $defaultAdminListenAddressDesc - name: "bootstrap-address" .}: ValidIpAddress + name: "bootstrap-address" .}: IpAddress bootstrapPort* {. desc: "The TCP/UDP port that will be used by the bootstrap node" @@ -194,9 +194,9 @@ type enrAddress* {. desc: "The public IP address of that ENR" - defaultValue: init(ValidIpAddress, defaultAdminListenAddress) + defaultValue: defaultAdminListenAddress defaultValueDesc: $defaultAdminListenAddressDesc - name: "enr-address" .}: ValidIpAddress + name: "enr-address" .}: IpAddress enrPort* {. desc: "The TCP/UDP port of that ENR" From 657de2721ab5203bef7cfdc69bcf56808f08fef4 Mon Sep 17 00:00:00 2001 From: tersec Date: Wed, 1 Nov 2023 14:00:21 +0100 Subject: [PATCH 16/35] fill blindedblockbody KZG commitments from deneb blinded header (#5550) --- beacon_chain/spec/datatypes/deneb.nim | 2 +- beacon_chain/validators/beacon_validators.nim | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/beacon_chain/spec/datatypes/deneb.nim b/beacon_chain/spec/datatypes/deneb.nim index b6d85b6365..5b0c735421 100644 --- a/beacon_chain/spec/datatypes/deneb.nim +++ b/beacon_chain/spec/datatypes/deneb.nim @@ -107,7 +107,7 @@ type executionPayload*: ExecutionPayload blockValue*: Wei kzgs*: KzgCommitments - proofs*:seq[KZGProof] + proofs*: seq[KZGProof] blobs*: Blobs # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/deneb/beacon-chain.md#executionpayloadheader diff --git a/beacon_chain/validators/beacon_validators.nim b/beacon_chain/validators/beacon_validators.nim index 3eeba31ea4..30afdaa927 100644 --- a/beacon_chain/validators/beacon_validators.nim +++ b/beacon_chain/validators/beacon_validators.nim @@ -619,6 +619,7 @@ from ./message_router_mev import func constructSignableBlindedBlock[T: capella_mev.SignedBlindedBeaconBlock]( blck: capella.BeaconBlock, executionPayloadHeader: capella.ExecutionPayloadHeader): T = + # Leaves signature field default, to be filled in by caller const blckFields = getFieldNames(typeof(blck)) blckBodyFields = getFieldNames(typeof(blck.body)) @@ -659,12 +660,16 @@ proc constructSignableBlindedBlock[T: deneb_mev.SignedBlindedBeaconBlockContents doAssert bbb.proofs.len == bbb.blob_roots.len doAssert bbb.proofs.len == bbb.commitments.len + assign(blindedBlock.message.body.blob_kzg_commitments, bbb.commitments) + + let blockRoot = hash_tree_root(blindedBlock.message) + if blindedBlockContents.signed_blinded_blob_sidecars.setLen(bbb.proofs.len): for i in 0 ..< blindedBlockContents.signed_blinded_blob_sidecars.lenu64: assign( blindedBlockContents.signed_blinded_blob_sidecars[i], deneb_mev.SignedBlindedBlobSidecar(message: deneb_mev.BlindedBlobSidecar( - block_root: hash_tree_root(blck), + block_root: blockRoot, index: i, slot: distinctBase(blck.slot), block_parent_root: blck.parent_root, From ab89e962655e9fb1aac5cc6d6ac41e032f40db9b Mon Sep 17 00:00:00 2001 From: tersec Date: Thu, 2 Nov 2023 04:56:04 +0100 Subject: [PATCH 17/35] fill in transactions root and KZG commitments from deneb.BlindedBeaconBlock in state transition (#5553) --- beacon_chain/spec/state_transition.nim | 16 +++++++++++----- beacon_chain/validators/beacon_validators.nim | 18 +++++++++++++----- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/beacon_chain/spec/state_transition.nim b/beacon_chain/spec/state_transition.nim index d74400115f..6b8cb9ded6 100644 --- a/beacon_chain/spec/state_transition.nim +++ b/beacon_chain/spec/state_transition.nim @@ -403,7 +403,8 @@ proc makeBeaconBlock*( # removed if we don't use invalid signatures there verificationFlags: UpdateFlags, transactions_root: Opt[Eth2Digest], - execution_payload_root: Opt[Eth2Digest]): + execution_payload_root: Opt[Eth2Digest], + kzg_commitments: Opt[KzgCommitments]): Result[ForkedBeaconBlock, cstring] = ## Create a block for the given state. The latest block applied to it will ## be used for the parent_root value, and the slot will be take from @@ -428,7 +429,7 @@ proc makeBeaconBlock*( rollback(state) return err(res.error()) - # Override for MEV + # Override for Builder API if transactions_root.isSome and execution_payload_root.isSome: withState(state): when consensusFork < ConsensusFork.Capella: @@ -455,6 +456,9 @@ proc makeBeaconBlock*( execution_payload_root.get, hash_tree_root(validator_changes.bls_to_execution_changes)]) elif consensusFork == ConsensusFork.Deneb: + forkyState.data.latest_execution_payload_header.transactions_root = + transactions_root.get + when executionPayload is deneb.ExecutionPayloadForSigning: # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/deneb/beacon-chain.md#beaconblockbody forkyState.data.latest_block_header.body_root = hash_tree_root( @@ -469,7 +473,7 @@ proc makeBeaconBlock*( hash_tree_root(sync_aggregate), execution_payload_root.get, hash_tree_root(validator_changes.bls_to_execution_changes), - hash_tree_root(executionPayload.kzgs) + hash_tree_root(kzg_commitments.get) ]) else: raiseAssert "Attempt to use non-Deneb payload with post-Deneb state" @@ -517,7 +521,8 @@ proc makeBeaconBlock*( attestations, deposits, validator_changes, sync_aggregate, executionPayload, rollback, cache, verificationFlags = {}, transactions_root = Opt.none Eth2Digest, - execution_payload_root = Opt.none Eth2Digest) + execution_payload_root = Opt.none Eth2Digest, + kzg_commitments = Opt.none KzgCommitments) proc makeBeaconBlock*( cfg: RuntimeConfig, state: var ForkedHashedBeaconState, @@ -536,4 +541,5 @@ proc makeBeaconBlock*( executionPayload, rollback, cache, verificationFlags = verificationFlags, transactions_root = Opt.none Eth2Digest, - execution_payload_root = Opt.none Eth2Digest) + execution_payload_root = Opt.none Eth2Digest, + kzg_commitments = Opt.none KzgCommitments) diff --git a/beacon_chain/validators/beacon_validators.nim b/beacon_chain/validators/beacon_validators.nim index 30afdaa927..7face4f0d2 100644 --- a/beacon_chain/validators/beacon_validators.nim +++ b/beacon_chain/validators/beacon_validators.nim @@ -440,7 +440,8 @@ proc makeBeaconBlockForHeadAndSlot*( execution_payload: Opt[PayloadType], transactions_root: Opt[Eth2Digest], execution_payload_root: Opt[Eth2Digest], - withdrawals_root: Opt[Eth2Digest]): + withdrawals_root: Opt[Eth2Digest], + kzg_commitments: Opt[KzgCommitments]): Future[ForkedBlockResult] {.async.} = # Advance state to the slot that we're proposing for var cache = StateCache() @@ -528,7 +529,8 @@ proc makeBeaconBlockForHeadAndSlot*( cache, verificationFlags = {}, transactions_root = transactions_root, - execution_payload_root = execution_payload_root).mapErr do (error: cstring) -> string: + execution_payload_root = execution_payload_root, + kzg_commitments = kzg_commitments).mapErr do (error: cstring) -> string: # This is almost certainly a bug, but it's complex enough that there's a # small risk it might happen even when most proposals succeed - thus we # log instead of asserting @@ -560,7 +562,8 @@ proc makeBeaconBlockForHeadAndSlot*( execution_payload = Opt.none(PayloadType), transactions_root = Opt.none(Eth2Digest), execution_payload_root = Opt.none(Eth2Digest), - withdrawals_root = Opt.none(Eth2Digest)) + withdrawals_root = Opt.none(Eth2Digest), + kzg_commitments = Opt.none(KzgCommitments)) proc getBlindedExecutionPayload[ EPH: capella.ExecutionPayloadHeader | @@ -846,6 +849,7 @@ proc getBlindedBlockParts[ template actualEPH: untyped = executionPayloadHeader.get.blindedBlckPart let withdrawals_root = Opt.some executionPayloadHeader.get.blindedBlckPart.withdrawals_root + const kzg_commitments = Opt.none KzgCommitments var shimExecutionPayload: PayloadType copyFields( @@ -855,7 +859,10 @@ proc getBlindedBlockParts[ type PayloadType = deneb.ExecutionPayloadForSigning template actualEPH: untyped = executionPayloadHeader.get.blindedBlckPart.execution_payload_header - let withdrawals_root = Opt.some actualEPH.withdrawals_root + let + withdrawals_root = Opt.some actualEPH.withdrawals_root + kzg_commitments = Opt.some( + executionPayloadHeader.get.blindedBlckPart.blinded_blobs_bundle.commitments) var shimExecutionPayload: PayloadType type DenebEPH = @@ -870,7 +877,8 @@ proc getBlindedBlockParts[ execution_payload = Opt.some shimExecutionPayload, transactions_root = Opt.some actualEPH.transactions_root, execution_payload_root = Opt.some hash_tree_root(actualEPH), - withdrawals_root = withdrawals_root) + withdrawals_root = withdrawals_root, + kzg_commitments = kzg_commitments) if newBlock.isErr(): # Haven't committed to the MEV block, so allow EL fallback. From baf81595a7fac13621923bd5bbe94c0704ee9421 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Fri, 3 Nov 2023 14:54:21 +0100 Subject: [PATCH 18/35] add `KZG_COMMITMENT_INCLUSION_PROOF_DEPTH` preset constant (#5555) `v1.4.0-beta.4` added `KZG_COMMITMENT_INCLUSION_PROOF_DEPTH` to preset: - Spec PR: https://github.com/ethereum/consensus-specs/pull/3531 - Gnosis PR: https://github.com/gnosischain/configs/pull/17 --- beacon_chain/rpc/rest_config_api.nim | 10 ++++++++++ beacon_chain/spec/presets/gnosis/altair_preset.nim | 2 +- beacon_chain/spec/presets/gnosis/bellatrix_preset.nim | 2 +- beacon_chain/spec/presets/gnosis/capella_preset.nim | 2 +- beacon_chain/spec/presets/gnosis/deneb_preset.nim | 6 ++++-- beacon_chain/spec/presets/gnosis/phase0_preset.nim | 2 +- beacon_chain/spec/presets/mainnet/deneb_preset.nim | 6 ++++-- beacon_chain/spec/presets/minimal/deneb_preset.nim | 6 ++++-- ncli/resttest-rules.json | 2 +- 9 files changed, 27 insertions(+), 11 deletions(-) diff --git a/beacon_chain/rpc/rest_config_api.nim b/beacon_chain/rpc/rest_config_api.nim index e01ec57b16..2b354909c3 100644 --- a/beacon_chain/rpc/rest_config_api.nim +++ b/beacon_chain/rpc/rest_config_api.nim @@ -128,6 +128,16 @@ proc installConfigApiHandlers*(router: var RestRouter, node: BeaconNode) = MAX_VALIDATORS_PER_WITHDRAWALS_SWEEP: Base10.toString(uint64(MAX_VALIDATORS_PER_WITHDRAWALS_SWEEP)), + # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.4/presets/mainnet/deneb.yaml + FIELD_ELEMENTS_PER_BLOB: + Base10.toString(deneb_preset.FIELD_ELEMENTS_PER_BLOB), + MAX_BLOB_COMMITMENTS_PER_BLOCK: + Base10.toString(MAX_BLOB_COMMITMENTS_PER_BLOCK), + MAX_BLOBS_PER_BLOCK: + Base10.toString(MAX_BLOBS_PER_BLOCK), + KZG_COMMITMENT_INCLUSION_PROOF_DEPTH: + Base10.toString(uint64(KZG_COMMITMENT_INCLUSION_PROOF_DEPTH)), + # https://github.com/ethereum/consensus-specs/blob/v1.3.0/configs/mainnet.yaml PRESET_BASE: cfg.PRESET_BASE, diff --git a/beacon_chain/spec/presets/gnosis/altair_preset.nim b/beacon_chain/spec/presets/gnosis/altair_preset.nim index 2db32470af..8841c8f412 100644 --- a/beacon_chain/spec/presets/gnosis/altair_preset.nim +++ b/beacon_chain/spec/presets/gnosis/altair_preset.nim @@ -6,7 +6,7 @@ # at your option. This file may not be copied, modified, or distributed except according to those terms. # Mainnet preset - Altair -# https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/presets/mainnet/altair.yaml +# https://github.com/gnosischain/configs/blob/b90374a1c63703db8235fcdb65aff2e909bc42b5/presets/gnosis/altair.yaml const # Updated penalty values # --------------------------------------------------------------- diff --git a/beacon_chain/spec/presets/gnosis/bellatrix_preset.nim b/beacon_chain/spec/presets/gnosis/bellatrix_preset.nim index 5375df014e..6b6fe445c3 100644 --- a/beacon_chain/spec/presets/gnosis/bellatrix_preset.nim +++ b/beacon_chain/spec/presets/gnosis/bellatrix_preset.nim @@ -6,7 +6,7 @@ # at your option. This file may not be copied, modified, or distributed except according to those terms. # Gnosis preset - Bellatrix -# https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/presets/mainnet/bellatrix.yaml +# https://github.com/gnosischain/configs/blob/b90374a1c63703db8235fcdb65aff2e909bc42b5/presets/gnosis/bellatrix.yaml const # Updated penalty values # --------------------------------------------------------------- diff --git a/beacon_chain/spec/presets/gnosis/capella_preset.nim b/beacon_chain/spec/presets/gnosis/capella_preset.nim index a61736e0ea..6807c16b61 100644 --- a/beacon_chain/spec/presets/gnosis/capella_preset.nim +++ b/beacon_chain/spec/presets/gnosis/capella_preset.nim @@ -6,7 +6,7 @@ # at your option. This file may not be copied, modified, or distributed except according to those terms. # Gnosis preset - Capella -# https://github.com/gnosischain/configs/blob/main/presets/gnosis/capella.yaml +# https://github.com/gnosischain/configs/blob/b90374a1c63703db8235fcdb65aff2e909bc42b5/presets/gnosis/capella.yaml const # Max operations per block # --------------------------------------------------------------- diff --git a/beacon_chain/spec/presets/gnosis/deneb_preset.nim b/beacon_chain/spec/presets/gnosis/deneb_preset.nim index 80aaffa2ac..3e34a87f51 100644 --- a/beacon_chain/spec/presets/gnosis/deneb_preset.nim +++ b/beacon_chain/spec/presets/gnosis/deneb_preset.nim @@ -5,8 +5,8 @@ # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). # at your option. This file may not be copied, modified, or distributed except according to those terms. -# Mainnet preset - Deneb -# https://github.com/ethereum/consensus-specs/blob/v1.4.0-alpha.1/presets/mainnet/deneb.yaml +# Gnosis preset - Deneb +# https://github.com/gnosischain/configs/blob/b90374a1c63703db8235fcdb65aff2e909bc42b5/presets/gnosis/deneb.yaml const # `uint64(4096)` FIELD_ELEMENTS_PER_BLOB*: uint64 = 4096 @@ -14,3 +14,5 @@ const MAX_BLOB_COMMITMENTS_PER_BLOCK*: uint64 = 4096 # `uint64(2**2)` (= 4) MAX_BLOBS_PER_BLOCK*: uint64 = 4 + # `floorlog2(get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments')) + 1 + ceillog2(MAX_BLOB_COMMITMENTS_PER_BLOCK)` = 4 + 1 + 12 = 17 + KZG_COMMITMENT_INCLUSION_PROOF_DEPTH* = 17 diff --git a/beacon_chain/spec/presets/gnosis/phase0_preset.nim b/beacon_chain/spec/presets/gnosis/phase0_preset.nim index 97d5f4cf24..cd6f9f7844 100644 --- a/beacon_chain/spec/presets/gnosis/phase0_preset.nim +++ b/beacon_chain/spec/presets/gnosis/phase0_preset.nim @@ -6,7 +6,7 @@ # at your option. This file may not be copied, modified, or distributed except according to those terms. # Gnosis preset - Phase0 -# https://github.com/ethereum/consensus-specs/blob/v1.4.0-alpha.1/presets/mainnet/phase0.yaml +# https://github.com/gnosischain/configs/blob/b90374a1c63703db8235fcdb65aff2e909bc42b5/presets/gnosis/phase0.yaml const # diff --git a/beacon_chain/spec/presets/mainnet/deneb_preset.nim b/beacon_chain/spec/presets/mainnet/deneb_preset.nim index 809b4623db..3caccbe8d5 100644 --- a/beacon_chain/spec/presets/mainnet/deneb_preset.nim +++ b/beacon_chain/spec/presets/mainnet/deneb_preset.nim @@ -6,11 +6,13 @@ # at your option. This file may not be copied, modified, or distributed except according to those terms. # Mainnet preset - Deneb -# https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/presets/mainnet/deneb.yaml +# https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.4/presets/mainnet/deneb.yaml const # `uint64(4096)` FIELD_ELEMENTS_PER_BLOB*: uint64 = 4096 # `uint64(2**12)` (= 4096) MAX_BLOB_COMMITMENTS_PER_BLOCK*: uint64 = 4096 - # `uint64(2**2)` (= 4) + # `uint64(6)` MAX_BLOBS_PER_BLOCK*: uint64 = 6 + # `floorlog2(get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments')) + 1 + ceillog2(MAX_BLOB_COMMITMENTS_PER_BLOCK)` = 4 + 1 + 12 = 17 + KZG_COMMITMENT_INCLUSION_PROOF_DEPTH* = 17 diff --git a/beacon_chain/spec/presets/minimal/deneb_preset.nim b/beacon_chain/spec/presets/minimal/deneb_preset.nim index d98d0a318a..6197ffac0d 100644 --- a/beacon_chain/spec/presets/minimal/deneb_preset.nim +++ b/beacon_chain/spec/presets/minimal/deneb_preset.nim @@ -6,11 +6,13 @@ # at your option. This file may not be copied, modified, or distributed except according to those terms. # Minimal preset - Deneb -# https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/presets/minimal/deneb.yaml +# https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.4/presets/minimal/deneb.yaml const - # [customized] + # `uint64(4096)` FIELD_ELEMENTS_PER_BLOB*: uint64 = 4096 # [customized] MAX_BLOB_COMMITMENTS_PER_BLOCK*: uint64 = 16 # `uint64(6)` MAX_BLOBS_PER_BLOCK*: uint64 = 6 + # [customized] `floorlog2(get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments')) + 1 + ceillog2(MAX_BLOB_COMMITMENTS_PER_BLOCK)` = 4 + 1 + 4 = 9 + KZG_COMMITMENT_INCLUSION_PROOF_DEPTH* = 9 diff --git a/ncli/resttest-rules.json b/ncli/resttest-rules.json index 59c249fee1..032bd98417 100644 --- a/ncli/resttest-rules.json +++ b/ncli/resttest-rules.json @@ -2826,7 +2826,7 @@ "response": { "status": {"operator": "equals", "value": "200"}, "headers": [{"key": "Content-Type", "value": "application/json", "operator": "equals"}], - "body": [{"operator": "jstructcmps", "start": ["data"], "value": {"MAX_COMMITTEES_PER_SLOT":"","TARGET_COMMITTEE_SIZE":"","MAX_VALIDATORS_PER_COMMITTEE":"","SHUFFLE_ROUND_COUNT":"","HYSTERESIS_QUOTIENT":"","HYSTERESIS_DOWNWARD_MULTIPLIER":"","HYSTERESIS_UPWARD_MULTIPLIER":"","MIN_DEPOSIT_AMOUNT":"","MAX_EFFECTIVE_BALANCE":"","EFFECTIVE_BALANCE_INCREMENT":"","MIN_ATTESTATION_INCLUSION_DELAY":"","SLOTS_PER_EPOCH":"","MIN_SEED_LOOKAHEAD":"","MAX_SEED_LOOKAHEAD":"","EPOCHS_PER_ETH1_VOTING_PERIOD":"","SLOTS_PER_HISTORICAL_ROOT":"","MIN_EPOCHS_TO_INACTIVITY_PENALTY":"","EPOCHS_PER_HISTORICAL_VECTOR":"","EPOCHS_PER_SLASHINGS_VECTOR":"","HISTORICAL_ROOTS_LIMIT":"","VALIDATOR_REGISTRY_LIMIT":"","BASE_REWARD_FACTOR":"","WHISTLEBLOWER_REWARD_QUOTIENT":"","PROPOSER_REWARD_QUOTIENT":"","INACTIVITY_PENALTY_QUOTIENT":"","MIN_SLASHING_PENALTY_QUOTIENT":"","PROPORTIONAL_SLASHING_MULTIPLIER":"","MAX_PROPOSER_SLASHINGS":"","MAX_ATTESTER_SLASHINGS":"","MAX_ATTESTATIONS":"","MAX_DEPOSITS":"","MAX_VOLUNTARY_EXITS":"","INACTIVITY_PENALTY_QUOTIENT_ALTAIR":"","MIN_SLASHING_PENALTY_QUOTIENT_ALTAIR":"","PROPORTIONAL_SLASHING_MULTIPLIER_ALTAIR":"","SYNC_COMMITTEE_SIZE":"","EPOCHS_PER_SYNC_COMMITTEE_PERIOD":"","MIN_SYNC_COMMITTEE_PARTICIPANTS":"","UPDATE_TIMEOUT":"","INACTIVITY_PENALTY_QUOTIENT_BELLATRIX":"","MIN_SLASHING_PENALTY_QUOTIENT_BELLATRIX":"","PROPORTIONAL_SLASHING_MULTIPLIER_BELLATRIX":"","MAX_BYTES_PER_TRANSACTION":"","MAX_TRANSACTIONS_PER_PAYLOAD":"","BYTES_PER_LOGS_BLOOM":"","MAX_EXTRA_DATA_BYTES":"","MAX_BLS_TO_EXECUTION_CHANGES":"","MAX_WITHDRAWALS_PER_PAYLOAD":"","MAX_VALIDATORS_PER_WITHDRAWALS_SWEEP":"","PRESET_BASE":"","CONFIG_NAME":"","TERMINAL_TOTAL_DIFFICULTY":"","TERMINAL_BLOCK_HASH":"","TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH":"","MIN_GENESIS_ACTIVE_VALIDATOR_COUNT":"","MIN_GENESIS_TIME":"","GENESIS_FORK_VERSION":"","GENESIS_DELAY":"","ALTAIR_FORK_VERSION":"","ALTAIR_FORK_EPOCH":"","BELLATRIX_FORK_VERSION":"","BELLATRIX_FORK_EPOCH":"","CAPELLA_FORK_VERSION":"","CAPELLA_FORK_EPOCH":"","DENEB_FORK_VERSION":"","DENEB_FORK_EPOCH":"","SECONDS_PER_SLOT":"","SECONDS_PER_ETH1_BLOCK":"","MIN_VALIDATOR_WITHDRAWABILITY_DELAY":"","SHARD_COMMITTEE_PERIOD":"","ETH1_FOLLOW_DISTANCE":"","INACTIVITY_SCORE_BIAS":"","INACTIVITY_SCORE_RECOVERY_RATE":"","EJECTION_BALANCE":"","MIN_PER_EPOCH_CHURN_LIMIT":"","CHURN_LIMIT_QUOTIENT":"","MAX_PER_EPOCH_ACTIVATION_CHURN_LIMIT":"","PROPOSER_SCORE_BOOST":"","DEPOSIT_CHAIN_ID":"","DEPOSIT_NETWORK_ID":"","DEPOSIT_CONTRACT_ADDRESS":"","BLS_WITHDRAWAL_PREFIX":"","ETH1_ADDRESS_WITHDRAWAL_PREFIX":"","DOMAIN_BEACON_PROPOSER":"","DOMAIN_BEACON_ATTESTER":"","DOMAIN_RANDAO":"","DOMAIN_DEPOSIT":"","DOMAIN_VOLUNTARY_EXIT":"","DOMAIN_SELECTION_PROOF":"","DOMAIN_AGGREGATE_AND_PROOF":"","TIMELY_SOURCE_FLAG_INDEX":"","TIMELY_TARGET_FLAG_INDEX":"","TIMELY_HEAD_FLAG_INDEX":"","TIMELY_SOURCE_WEIGHT":"","TIMELY_TARGET_WEIGHT":"","TIMELY_HEAD_WEIGHT":"","SYNC_REWARD_WEIGHT":"","PROPOSER_WEIGHT":"","WEIGHT_DENOMINATOR":"","DOMAIN_SYNC_COMMITTEE":"","DOMAIN_SYNC_COMMITTEE_SELECTION_PROOF":"","DOMAIN_CONTRIBUTION_AND_PROOF":"","DOMAIN_BLS_TO_EXECUTION_CHANGE":"","TARGET_AGGREGATORS_PER_COMMITTEE":"","ATTESTATION_SUBNET_COUNT":"","TARGET_AGGREGATORS_PER_SYNC_SUBCOMMITTEE":"","SYNC_COMMITTEE_SUBNET_COUNT":""}}] + "body": [{"operator": "jstructcmps", "start": ["data"], "value": {"MAX_COMMITTEES_PER_SLOT":"","TARGET_COMMITTEE_SIZE":"","MAX_VALIDATORS_PER_COMMITTEE":"","SHUFFLE_ROUND_COUNT":"","HYSTERESIS_QUOTIENT":"","HYSTERESIS_DOWNWARD_MULTIPLIER":"","HYSTERESIS_UPWARD_MULTIPLIER":"","MIN_DEPOSIT_AMOUNT":"","MAX_EFFECTIVE_BALANCE":"","EFFECTIVE_BALANCE_INCREMENT":"","MIN_ATTESTATION_INCLUSION_DELAY":"","SLOTS_PER_EPOCH":"","MIN_SEED_LOOKAHEAD":"","MAX_SEED_LOOKAHEAD":"","EPOCHS_PER_ETH1_VOTING_PERIOD":"","SLOTS_PER_HISTORICAL_ROOT":"","MIN_EPOCHS_TO_INACTIVITY_PENALTY":"","EPOCHS_PER_HISTORICAL_VECTOR":"","EPOCHS_PER_SLASHINGS_VECTOR":"","HISTORICAL_ROOTS_LIMIT":"","VALIDATOR_REGISTRY_LIMIT":"","BASE_REWARD_FACTOR":"","WHISTLEBLOWER_REWARD_QUOTIENT":"","PROPOSER_REWARD_QUOTIENT":"","INACTIVITY_PENALTY_QUOTIENT":"","MIN_SLASHING_PENALTY_QUOTIENT":"","PROPORTIONAL_SLASHING_MULTIPLIER":"","MAX_PROPOSER_SLASHINGS":"","MAX_ATTESTER_SLASHINGS":"","MAX_ATTESTATIONS":"","MAX_DEPOSITS":"","MAX_VOLUNTARY_EXITS":"","INACTIVITY_PENALTY_QUOTIENT_ALTAIR":"","MIN_SLASHING_PENALTY_QUOTIENT_ALTAIR":"","PROPORTIONAL_SLASHING_MULTIPLIER_ALTAIR":"","SYNC_COMMITTEE_SIZE":"","EPOCHS_PER_SYNC_COMMITTEE_PERIOD":"","MIN_SYNC_COMMITTEE_PARTICIPANTS":"","UPDATE_TIMEOUT":"","INACTIVITY_PENALTY_QUOTIENT_BELLATRIX":"","MIN_SLASHING_PENALTY_QUOTIENT_BELLATRIX":"","PROPORTIONAL_SLASHING_MULTIPLIER_BELLATRIX":"","MAX_BYTES_PER_TRANSACTION":"","MAX_TRANSACTIONS_PER_PAYLOAD":"","BYTES_PER_LOGS_BLOOM":"","MAX_EXTRA_DATA_BYTES":"","MAX_BLS_TO_EXECUTION_CHANGES":"","MAX_WITHDRAWALS_PER_PAYLOAD":"","MAX_VALIDATORS_PER_WITHDRAWALS_SWEEP":"","PRESET_BASE":"","CONFIG_NAME":"","TERMINAL_TOTAL_DIFFICULTY":"","TERMINAL_BLOCK_HASH":"","TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH":"","MIN_GENESIS_ACTIVE_VALIDATOR_COUNT":"","MIN_GENESIS_TIME":"","GENESIS_FORK_VERSION":"","GENESIS_DELAY":"","ALTAIR_FORK_VERSION":"","ALTAIR_FORK_EPOCH":"","BELLATRIX_FORK_VERSION":"","BELLATRIX_FORK_EPOCH":"","CAPELLA_FORK_VERSION":"","CAPELLA_FORK_EPOCH":"","DENEB_FORK_VERSION":"","DENEB_FORK_EPOCH":"","SECONDS_PER_SLOT":"","SECONDS_PER_ETH1_BLOCK":"","MIN_VALIDATOR_WITHDRAWABILITY_DELAY":"","FIELD_ELEMENTS_PER_BLOB":"","MAX_BLOB_COMMITMENTS_PER_BLOCK":"","MAX_BLOBS_PER_BLOCK":"","KZG_COMMITMENT_INCLUSION_PROOF_DEPTH":"","SHARD_COMMITTEE_PERIOD":"","ETH1_FOLLOW_DISTANCE":"","INACTIVITY_SCORE_BIAS":"","INACTIVITY_SCORE_RECOVERY_RATE":"","EJECTION_BALANCE":"","MIN_PER_EPOCH_CHURN_LIMIT":"","CHURN_LIMIT_QUOTIENT":"","MAX_PER_EPOCH_ACTIVATION_CHURN_LIMIT":"","PROPOSER_SCORE_BOOST":"","DEPOSIT_CHAIN_ID":"","DEPOSIT_NETWORK_ID":"","DEPOSIT_CONTRACT_ADDRESS":"","BLS_WITHDRAWAL_PREFIX":"","ETH1_ADDRESS_WITHDRAWAL_PREFIX":"","DOMAIN_BEACON_PROPOSER":"","DOMAIN_BEACON_ATTESTER":"","DOMAIN_RANDAO":"","DOMAIN_DEPOSIT":"","DOMAIN_VOLUNTARY_EXIT":"","DOMAIN_SELECTION_PROOF":"","DOMAIN_AGGREGATE_AND_PROOF":"","TIMELY_SOURCE_FLAG_INDEX":"","TIMELY_TARGET_FLAG_INDEX":"","TIMELY_HEAD_FLAG_INDEX":"","TIMELY_SOURCE_WEIGHT":"","TIMELY_TARGET_WEIGHT":"","TIMELY_HEAD_WEIGHT":"","SYNC_REWARD_WEIGHT":"","PROPOSER_WEIGHT":"","WEIGHT_DENOMINATOR":"","DOMAIN_SYNC_COMMITTEE":"","DOMAIN_SYNC_COMMITTEE_SELECTION_PROOF":"","DOMAIN_CONTRIBUTION_AND_PROOF":"","DOMAIN_BLS_TO_EXECUTION_CHANGE":"","TARGET_AGGREGATORS_PER_COMMITTEE":"","ATTESTATION_SUBNET_COUNT":"","TARGET_AGGREGATORS_PER_SYNC_SUBCOMMITTEE":"","SYNC_COMMITTEE_SUBNET_COUNT":""}}] } }, { From c95f9feec435313f866a942543bd2b1fcddf5666 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Fri, 3 Nov 2023 15:37:44 +0100 Subject: [PATCH 19/35] remove unused Deneb code (#5556) The `process_blob_kzg_commitments` step was removed in `v1.4.0-alpha.1`, but we haven't deleted the now unused functions. Do that now. --- beacon_chain/spec/datatypes/deneb.nim | 3 -- beacon_chain/spec/state_transition_block.nim | 54 -------------------- 2 files changed, 57 deletions(-) diff --git a/beacon_chain/spec/datatypes/deneb.nim b/beacon_chain/spec/datatypes/deneb.nim index 5b0c735421..250314df48 100644 --- a/beacon_chain/spec/datatypes/deneb.nim +++ b/beacon_chain/spec/datatypes/deneb.nim @@ -33,9 +33,6 @@ const # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.2/specs/deneb/polynomial-commitments.md#constants BYTES_PER_FIELD_ELEMENT = 32 - # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/deneb/beacon-chain.md#blob - BLOB_TX_TYPE* = 0x03'u8 - # https://github.com/ethereum/consensus-specs/blob/v1.3.0/specs/deneb/polynomial-commitments.md#constants BLS_MODULUS* = "52435875175126190479447740508185965837690552500527637822603658699938581184513".u256 diff --git a/beacon_chain/spec/state_transition_block.nim b/beacon_chain/spec/state_transition_block.nim index 21b027e565..1d0270ee5e 100644 --- a/beacon_chain/spec/state_transition_block.nim +++ b/beacon_chain/spec/state_transition_block.nim @@ -715,35 +715,6 @@ func process_withdrawals*( ok() -# https://github.com/ethereum/consensus-specs/blob/v1.3.0/specs/deneb/beacon-chain.md#tx_peek_blob_versioned_hashes -func tx_peek_blob_versioned_hashes(opaque_tx: Transaction): - Result[seq[VersionedHash], cstring] = - ## This function retrieves the hashes from the `SignedBlobTransaction` as - ## defined in Deneb, using SSZ offsets. Offsets are little-endian `uint32` - ## values, as defined in the SSZ specification. See the full details of - ## `blob_versioned_hashes` offset calculation. - if not (opaque_tx[0] == BLOB_TX_TYPE): - return err("tx_peek_blob_versioned_hashes: invalid opaque transaction type") - let message_offset = 1 + bytes_to_uint32(opaque_tx.asSeq.toOpenArray(1, 4)) - - if opaque_tx.lenu64 < (message_offset + 192).uint64: - return err("tx_peek_blob_versioned_hashes: opaque transaction too short") - - # field offset: 32 + 8 + 32 + 32 + 8 + 4 + 32 + 4 + 4 + 32 = 188 - let blob_versioned_hashes_offset = ( - message_offset + bytes_to_uint32( - opaque_tx[(message_offset + 188) ..< (message_offset + 192)])) - - if blob_versioned_hashes_offset.uint64 > high(int).uint64: - return err("tx_peek_blob_versioned_hashes: blob_versioned_hashes_offset too high") - - var res: seq[VersionedHash] - for x in countup(blob_versioned_hashes_offset.int, len(opaque_tx) - 1, 32): - var versionedHash: VersionedHash - versionedHash[0 .. 31] = opaque_tx.asSeq.toOpenArray(x, x + 31) - res.add versionedHash - ok res - # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/deneb/beacon-chain.md#kzg_commitment_to_versioned_hash func kzg_commitment_to_versioned_hash*( kzg_commitment: KzgCommitment): VersionedHash = @@ -755,31 +726,6 @@ func kzg_commitment_to_versioned_hash*( res[1 .. 31] = eth2digest(kzg_commitment).data.toOpenArray(1, 31) res -# https://github.com/ethereum/consensus-specs/blob/v1.3.0/specs/deneb/beacon-chain.md#verify_kzg_commitments_against_transactions -func verify_kzg_commitments_against_transactions*( - transactions: seq[Transaction], - kzg_commitments: seq[KzgCommitment]): bool = - var all_versioned_hashes: seq[VersionedHash] - for tx in transactions: - if tx[0] == BLOB_TX_TYPE: - let maybe_versioned_hashed = tx_peek_blob_versioned_hashes(tx) - if maybe_versioned_hashed.isErr: - return false - all_versioned_hashes.add maybe_versioned_hashed.get - # TODO valueOr version fails to compile - #all_versioned_hashes.add tx_peek_blob_versioned_hashes(tx).valueOr: - # return false - - all_versioned_hashes == mapIt( - kzg_commitments, it.kzg_commitment_to_versioned_hash) - -func process_blob_kzg_commitments( - body: deneb.BeaconBlockBody | deneb.TrustedBeaconBlockBody | - deneb.SigVerifiedBeaconBlockBody): bool = - verify_kzg_commitments_against_transactions( - body.execution_payload.transactions.asSeq, - body.blob_kzg_commitments.asSeq) - # https://github.com/ethereum/consensus-specs/blob/v1.3.0/specs/deneb/fork-choice.md#validate_blobs proc validate_blobs*(expected_kzg_commitments: seq[KzgCommitment], blobs: seq[KzgBlob], From 29fe9589089e2ee833b7023c323626f7a5116597 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Fri, 3 Nov 2023 16:07:49 +0100 Subject: [PATCH 20/35] allow to pass trusted node sync options during BN startup (#5545) Using trusted node sync currently requires to run two commands - first the `trustedNodeSync` command to initialize the database, followed by the regular startup command to continue syncing. The `trustedNodeSync` options are now also available during regular startup, and are used when the database is empty to simplify setting up a new Nimbus beacon node. This also aligns behaviour closer with other Ethereum consensus implementations. The new logic only applies if the database has not yet been initialized; same as before. Also, the database needs to be removed under the same conditions as before when a fresh sync is desired. --- beacon_chain/conf.nim | 17 +++++++-- beacon_chain/nimbus_beacon_node.nim | 20 ++++++++++ docs/the_nimbus_book/src/options.md | 8 +++- docs/the_nimbus_book/src/start-syncing.md | 46 +++++++++++++++++++++++ 4 files changed, 87 insertions(+), 4 deletions(-) diff --git a/beacon_chain/conf.nim b/beacon_chain/conf.nim index 0692c2e091..274c443d18 100644 --- a/beacon_chain/conf.nim +++ b/beacon_chain/conf.nim @@ -339,16 +339,27 @@ type desc: "Weak subjectivity checkpoint in the format block_root:epoch_number" name: "weak-subjectivity-checkpoint" .}: Option[Checkpoint] + externalBeaconApiUrl* {. + desc: "External beacon API to use for syncing (on empty database)" + name: "external-beacon-api-url" .}: Option[string] + syncLightClient* {. - desc: "Accelerate execution layer sync using light client" + desc: "Accelerate sync using light client" defaultValue: true name: "sync-light-client" .}: bool trustedBlockRoot* {. - hidden - desc: "Recent trusted finalized block root to initialize light client from" + desc: "Recent trusted finalized block root to sync from external " & + "beacon API (with `--external-beacon-api-url`). " & + "Uses the light client sync protocol to obtain the latest " & + "finalized checkpoint (LC is initialized from trusted block root)" name: "trusted-block-root" .}: Option[Eth2Digest] + trustedStateRoot* {. + desc: "Recent trusted finalized state root to sync from external " & + "beacon API (with `--external-beacon-api-url`)" + name: "trusted-state-root" .}: Option[Eth2Digest] + finalizedCheckpointState* {. desc: "SSZ file specifying a recent finalized state" name: "finalized-checkpoint-state" .}: Option[InputFile] diff --git a/beacon_chain/nimbus_beacon_node.nim b/beacon_chain/nimbus_beacon_node.nim index cb49eb300d..1721505eca 100644 --- a/beacon_chain/nimbus_beacon_node.nim +++ b/beacon_chain/nimbus_beacon_node.nim @@ -555,6 +555,26 @@ proc init*(T: type BeaconNode, ) db = BeaconChainDB.new(config.databaseDir, cfg, inMemory = false) + if config.externalBeaconApiUrl.isSome and ChainDAGRef.isInitialized(db).isErr: + if config.trustedStateRoot.isNone and config.trustedBlockRoot.isNone: + warn "Ignoring `--external-beacon-api-url`, neither " & + "`--trusted-block-root` nor `--trusted-state-root` are provided", + externalBeaconApiUrl = config.externalBeaconApiUrl.get, + trustedBlockRoot = config.trustedBlockRoot, + trustedStateRoot = config.trustedStateRoot + else: + await db.doRunTrustedNodeSync( + metadata, + config.databaseDir, + config.eraDir, + config.externalBeaconApiUrl.get, + config.trustedStateRoot.map do (x: Eth2Digest) -> string: + "0x" & x.data.toHex, + config.trustedBlockRoot, + backfill = false, + reindex = false, + downloadDepositSnapshot = false) + if config.finalizedCheckpointBlock.isSome: warn "--finalized-checkpoint-block has been deprecated, ignoring" diff --git a/docs/the_nimbus_book/src/options.md b/docs/the_nimbus_book/src/options.md index 03c0709263..914ea86bb2 100644 --- a/docs/the_nimbus_book/src/options.md +++ b/docs/the_nimbus_book/src/options.md @@ -66,7 +66,13 @@ The following options are available: seen by other nodes it communicates with. This option allows to enable/disable this functionality [=false]. --weak-subjectivity-checkpoint Weak subjectivity checkpoint in the format block_root:epoch_number. - --sync-light-client Accelerate execution layer sync using light client [=true]. + --external-beacon-api-url External beacon API to use for syncing (on empty database). + --sync-light-client Accelerate sync using light client [=true]. + --trusted-block-root Recent trusted finalized block root to sync from external beacon API (with + `--external-beacon-api-url`). Uses the light client sync protocol to obtain the + latest finalized checkpoint (LC is initialized from trusted block root). + --trusted-state-root Recent trusted finalized state root to sync from external beacon API (with + `--external-beacon-api-url`). --finalized-checkpoint-state SSZ file specifying a recent finalized state. --genesis-state SSZ file specifying the genesis state of the network (for networks without a built-in genesis state). diff --git a/docs/the_nimbus_book/src/start-syncing.md b/docs/the_nimbus_book/src/start-syncing.md index e3be4916a2..8ae1f5cc8b 100644 --- a/docs/the_nimbus_book/src/start-syncing.md +++ b/docs/the_nimbus_book/src/start-syncing.md @@ -82,3 +82,49 @@ More information is available from the [options](./options.md) page. ## Keep track of your sync progress See [here](./keep-an-eye.md#keep-track-of-your-syncing-progress) for how to keep track of your sync progress. + +## Checkpoint sync + +!!! note "" + This feature is available from `v23.11.0` onwards. + +You can use an existing synced node or a third-party service to accelerate sync significantly. Instead of downloading and verifying the entire blockchain, you can point Nimbus to a trusted block. + +!!! warning + Selecting a block from an untrusted source or using an outdated block or state may lead to Nimbus syncing to an unexpected state. Especially when [running a validator](./run-a-validator.md), it is vital to pick a recent trusted block for checkpoint sync, and to verify that Nimbus is synced to the correct state before starting validator duties. + +!!! note + The Nimbus database must be empty to use checkpoint sync. When using a pre-existing database, checkpoint sync options are ignored. + +!!! tip + A list of community-operated checkpoint sync nodes can be found [here](https://eth-clients.github.io/checkpoint-sync-endpoints/). + +To use checkpoint sync, run the following commands (inserting the checkpoint sync endpoint and your own trusted block root): + +=== "Holesky" + ```sh + CHECKPOINT_SYNC_ENDPOINT=http://127.0.0.1:8551 + TRUSTED_BLOCK_ROOT=0x1234567890123456789012345678901234567890123456789012345678901234 + ./run-holesky-beacon-node.sh \ + --external-beacon-api-url=$CHECKPOINT_SYNC_ENDPOINT \ + --trusted-block-root=$TRUSTED_BLOCK_ROOT + ``` + +=== "Mainnet" + ```sh + TRUSTED_BLOCK_ROOT=0x1234567890123456789012345678901234567890123456789012345678901234 + ./run-mainnet-beacon-node.sh \ + --external-beacon-api-url=$CHECKPOINT_SYNC_ENDPOINT \ + --trusted-block-root=$TRUSTED_BLOCK_ROOT + ``` + +The following [configuration options](./options.md) control checkpoint sync behaviour: + +| Option | Description | +|------------------------------------------|-------------| +| `--external-beacon-api-url` |
  • External beacon API to use for checkpoint sync
| +| `--trusted-block-root` |
  • Recent trusted finalized block root to sync from external beacon API
  • Uses the [light client sync protocol](https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/altair/light-client/sync-protocol.md) to obtain the latest finalized checkpoint
| +| `--trusted-state-root` |
  • Recent trusted finalized state root to sync from external beacon API
  • Takes precedence over `--trusted-block-root` if both are specified
| + +!!! info + If the external beacon API does not support serving [light client data](./light-client-data.md), use the `--trusted-state-root` option instead of `--trusted-block-root`. From 8cec3af61c8847675235c69c3e716567bc3951e3 Mon Sep 17 00:00:00 2001 From: Eugene Kabanov Date: Sat, 4 Nov 2023 09:14:14 +0200 Subject: [PATCH 21/35] VC: Obtain randao signature before slot proposal. (#5490) * Randao calculation caching for VC implementation. * Add time monitoring for randao signatures process. * Add delay calculation. * Address review comments. * Address review comments. --- .../validator_client/block_service.nim | 103 +++++++++++++----- beacon_chain/validator_client/common.nim | 3 +- beacon_chain/validators/validator_pool.nim | 15 ++- 3 files changed, 94 insertions(+), 27 deletions(-) diff --git a/beacon_chain/validator_client/block_service.nim b/beacon_chain/validator_client/block_service.nim index ad4cad0573..995d3c377e 100644 --- a/beacon_chain/validator_client/block_service.nim +++ b/beacon_chain/validator_client/block_service.nim @@ -32,6 +32,9 @@ type blockRoot*: Eth2Digest data*: ForkedBlindedBeaconBlock +proc proposeBlock(vc: ValidatorClientRef, slot: Slot, + proposerKey: ValidatorPubKey) {.async.} + proc produceBlock( vc: ValidatorClientRef, currentSlot, slot: Slot, @@ -86,7 +89,6 @@ proc produceBlock( data: ForkedBeaconBlock.init(blck), blobsOpt: Opt.some(blobs))) - proc produceBlindedBlock( vc: ValidatorClientRef, currentSlot, slot: Slot, @@ -125,6 +127,58 @@ proc lazyWait[T](fut: Future[T]) {.async.} = except CatchableError: discard +proc prepareRandao(vc: ValidatorClientRef, slot: Slot, + proposerKey: ValidatorPubKey) {.async.} = + if slot == GENESIS_SLOT: + return + + let + destSlot = slot - 1'u64 + destOffset = TimeDiff(nanoseconds: NANOSECONDS_PER_SLOT.int64 div 2) + deadline = destSlot.start_beacon_time() + destOffset + epoch = slot.epoch() + # We going to wait to T - (T / 4 * 2), where T is proposer's + # duty slot. + currentSlot = (await vc.checkedWaitForSlot(destSlot, destOffset, + false)).valueOr: + debug "Unable to perform RANDAO signature preparation because of " & + "system time failure" + return + validator = + vc.getValidatorForDuties(proposerKey, slot, true).valueOr: return + + if currentSlot <= destSlot: + # We do not need result, because we want it to be cached. + let + start = Moment.now() + genesisRoot = vc.beaconGenesis.genesis_validators_root + fork = vc.forkAtEpoch(epoch) + rsig = await validator.getEpochSignature(fork, genesisRoot, epoch) + timeElapsed = Moment.now() - start + if rsig.isErr(): + debug "Unable to prepare RANDAO signature", epoch = epoch, + validator = shortLog(validator), elapsed_time = timeElapsed, + current_slot = currentSlot, destination_slot = destSlot, + delay = vc.getDelay(deadline) + else: + debug "RANDAO signature has been prepared", epoch = epoch, + validator = shortLog(validator), elapsed_time = timeElapsed, + current_slot = currentSlot, destination_slot = destSlot, + delay = vc.getDelay(deadline) + else: + debug "RANDAO signature preparation timed out", epoch = epoch, + validator = shortLog(validator), + current_slot = currentSlot, destination_slot = destSlot, + delay = vc.getDelay(deadline) + +proc spawnProposalTask(vc: ValidatorClientRef, + duty: RestProposerDuty): ProposerTask = + ProposerTask( + randaoFut: prepareRandao(vc, duty.slot, duty.pubkey), + proposeFut: proposeBlock(vc, duty.slot, duty.pubkey), + duty: duty + ) + proc publishBlock(vc: ValidatorClientRef, currentSlot, slot: Slot, validator: AttachedValidator) {.async.} = let @@ -146,21 +200,22 @@ proc publishBlock(vc: ValidatorClientRef, currentSlot, slot: Slot, debug "Publishing block", delay = vc.getDelay(slot.block_deadline()), genesis_root = genesisRoot, graffiti = graffiti, fork = fork - let randaoReveal = - try: - let res = await validator.getEpochSignature(fork, genesisRoot, slot.epoch) - if res.isErr(): - warn "Unable to generate randao reveal using remote signer", - reason = res.error() + + let + randaoReveal = + try: + (await validator.getEpochSignature(fork, genesisRoot, + slot.epoch())).valueOr: + warn "Unable to generate RANDAO reveal using remote signer", + reason = error + return + except CancelledError as exc: + debug "RANDAO reveal production has been interrupted" + raise exc + except CatchableError as exc: + error "An unexpected error occurred while receiving RANDAO data", + error_name = exc.name, error_msg = exc.msg return - res.get() - except CancelledError as exc: - debug "Randao reveal production has been interrupted" - raise exc - except CatchableError as exc: - error "An unexpected error occurred while receiving randao data", - error_name = exc.name, error_msg = exc.msg - return var beaconBlocks = block: @@ -408,11 +463,6 @@ proc proposeBlock(vc: ValidatorClientRef, slot: Slot, error "Unexpected error encountered while proposing block", slot = slot, validator = shortLog(validator) -proc spawnProposalTask(vc: ValidatorClientRef, - duty: RestProposerDuty): ProposerTask = - let future = proposeBlock(vc, duty.slot, duty.pubkey) - ProposerTask(future: future, duty: duty) - proc contains(data: openArray[RestProposerDuty], task: ProposerTask): bool = for item in data: if (item.pubkey == task.duty.pubkey) and (item.slot == task.duty.slot): @@ -462,13 +512,14 @@ proc addOrReplaceProposers*(vc: ValidatorClientRef, epoch: Epoch, for task in epochDuties.duties: if task notin duties: # Task is no more relevant, so cancel it. - debug "Cancelling running proposal duty task", + debug "Cancelling running proposal duty tasks", slot = task.duty.slot, validator = shortLog(task.duty.pubkey) - task.future.cancelSoon() + task.proposeFut.cancelSoon() + task.randaoFut.cancelSoon() else: # If task is already running for proper slot, we keep it alive. - debug "Keep running previous proposal duty task", + debug "Keep running previous proposal duty tasks", slot = task.duty.slot, validator = shortLog(task.duty.pubkey) res.add(task) @@ -783,8 +834,10 @@ proc mainLoop(service: BlockServiceRef) {.async.} = var res: seq[FutureBase] for epoch, data in vc.proposers.pairs(): for duty in data.duties.items(): - if not(duty.future.finished()): - res.add(duty.future.cancelAndWait()) + if not(duty.proposeFut.finished()): + res.add(duty.proposeFut.cancelAndWait()) + if not(duty.randaoFut.finished()): + res.add(duty.randaoFut.cancelAndWait()) await noCancel allFutures(res) proc init*(t: typedesc[BlockServiceRef], diff --git a/beacon_chain/validator_client/common.nim b/beacon_chain/validator_client/common.nim index 7202bc9aaf..eb4a505571 100644 --- a/beacon_chain/validator_client/common.nim +++ b/beacon_chain/validator_client/common.nim @@ -95,7 +95,8 @@ type ProposerTask* = object duty*: RestProposerDuty - future*: Future[void] + proposeFut*: Future[void] + randaoFut*: Future[void] ProposedData* = object epoch*: Epoch diff --git a/beacon_chain/validators/validator_pool.nim b/beacon_chain/validators/validator_pool.nim index 345712e475..6ad6e4416d 100644 --- a/beacon_chain/validators/validator_pool.nim +++ b/beacon_chain/validators/validator_pool.nim @@ -66,6 +66,10 @@ type # if the validator will be aggregating (in the near future) slotSignature*: Opt[tuple[slot: Slot, signature: ValidatorSig]] + # Cache the latest epoch signature - the epoch signature is used for block + # proposing. + epochSignature*: Opt[tuple[epoch: Epoch, signature: ValidatorSig]] + # For the external payload builder; each epoch, the external payload # builder should be informed of current validators externalBuilderRegistration*: Opt[SignedValidatorRegistrationV1] @@ -746,7 +750,10 @@ proc getContributionAndProofSignature*(v: AttachedValidator, fork: Fork, proc getEpochSignature*(v: AttachedValidator, fork: Fork, genesis_validators_root: Eth2Digest, epoch: Epoch ): Future[SignatureResult] {.async.} = - return + if v.epochSignature.isSome and v.epochSignature.get.epoch == epoch: + return SignatureResult.ok(v.epochSignature.get.signature) + + let signature = case v.kind of ValidatorKind.Local: SignatureResult.ok(get_epoch_signature( @@ -757,6 +764,12 @@ proc getEpochSignature*(v: AttachedValidator, fork: Fork, fork, genesis_validators_root, epoch) await v.signData(request) + if signature.isErr: + return signature + + v.epochSignature = Opt.some((epoch, signature.get)) + signature + # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/phase0/validator.md#aggregation-selection proc getSlotSignature*(v: AttachedValidator, fork: Fork, genesis_validators_root: Eth2Digest, slot: Slot From 3e9b3a50d0c6e53f7419803b3dc4d9db5a39bcc8 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Sat, 4 Nov 2023 09:01:31 +0100 Subject: [PATCH 22/35] libp2p: revert scoring (#5549) * Use libp2p b2eac7e-and-revert-c6aa085 * Fix libp2p branch reference --------- Co-authored-by: Diego Co-authored-by: Zahary Karadjov --- .gitmodules | 2 +- vendor/nim-libp2p | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 71d512a1ca..c0e39c3a25 100644 --- a/.gitmodules +++ b/.gitmodules @@ -24,7 +24,7 @@ path = vendor/nim-libp2p url = https://github.com/status-im/nim-libp2p.git ignore = untracked - branch = unstable + branch = b2eac7e-and-revert-c6aa085 [submodule "vendor/nimbus-build-system"] path = vendor/nimbus-build-system url = https://github.com/status-im/nimbus-build-system.git diff --git a/vendor/nim-libp2p b/vendor/nim-libp2p index b2eac7ecbd..0011a5e720 160000 --- a/vendor/nim-libp2p +++ b/vendor/nim-libp2p @@ -1 +1 @@ -Subproject commit b2eac7ecbdb695b0b7033f2069b03a63d28aee2b +Subproject commit 0011a5e720e2e61beb75b3384cd788c72548284a From fb18f09e5527d912b222461d1c15b639ae249ca7 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Sat, 4 Nov 2023 10:25:06 +0100 Subject: [PATCH 23/35] align `Blobs` limit for SSZ transport (#5558) Capacity should be set to theoretical limit to ensure correct hash root. Actual length may be shorter. Only use is `ExecutionPayloadForSigning` so it doesn't matter yet in practice, but still worth fixing. --- beacon_chain/spec/datatypes/deneb.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_chain/spec/datatypes/deneb.nim b/beacon_chain/spec/datatypes/deneb.nim index 250314df48..f58a8accec 100644 --- a/beacon_chain/spec/datatypes/deneb.nim +++ b/beacon_chain/spec/datatypes/deneb.nim @@ -38,7 +38,7 @@ const type KzgCommitments* = List[KzgCommitment, Limit MAX_BLOB_COMMITMENTS_PER_BLOCK] - Blobs* = List[Blob, Limit MAX_BLOBS_PER_BLOCK] + Blobs* = List[Blob, Limit MAX_BLOB_COMMITMENTS_PER_BLOCK] # TODO this apparently is suppposed to be SSZ-equivalent to Bytes32, but # current spec doesn't ever SSZ-serialize it or hash_tree_root it From a05278e26374fa640dfb907281489442b9c6b373 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Sat, 4 Nov 2023 14:49:58 +0100 Subject: [PATCH 24/35] wrap `kzgs`/`proofs`/`blobs` fields as `BlobsBundle` (#5562) Less type conversion / copying by keeping the `BlobsBundle` together. --- beacon_chain/el/el_manager.nim | 11 ++++--- beacon_chain/rpc/rest_validator_api.nim | 33 ++++++------------- beacon_chain/spec/datatypes/deneb.nim | 17 +++++----- beacon_chain/spec/state_transition.nim | 2 +- beacon_chain/sync/request_manager.nim | 5 ++- beacon_chain/validators/beacon_validators.nim | 22 ++++--------- 6 files changed, 34 insertions(+), 56 deletions(-) diff --git a/beacon_chain/el/el_manager.nim b/beacon_chain/el/el_manager.nim index e9334594f7..271afc47b0 100644 --- a/beacon_chain/el/el_manager.nim +++ b/beacon_chain/el/el_manager.nim @@ -551,10 +551,13 @@ func asConsensusType*(payload: engine_api.GetPayloadV3Response): # The `mapIt` calls below are necessary only because we use different distinct # types for KZG commitments and Blobs in the `web3` and the `deneb` spec types. # Both are defined as `array[N, byte]` under the hood. - kzgs: KzgCommitments payload.blobsBundle.commitments.mapIt(it.bytes), - proofs: payload.blobsBundle.proofs.mapIt(it.bytes), - blobs: Blobs payload.blobsBundle.blobs.mapIt(it.bytes) - ) + blobsBundle: BlobsBundle( + commitments: KzgCommitments.init( + payload.blobsBundle.commitments.mapIt(it.bytes)), + proofs: KzgProofs.init( + payload.blobsBundle.proofs.mapIt(it.bytes)), + blobs: Blobs.init( + payload.blobsBundle.blobs.mapIt(it.bytes)))) func asEngineExecutionPayload*(executionPayload: bellatrix.ExecutionPayload): ExecutionPayloadV1 = diff --git a/beacon_chain/rpc/rest_validator_api.nim b/beacon_chain/rpc/rest_validator_api.nim index 4312ff0410..c016f899f7 100644 --- a/beacon_chain/rpc/rest_validator_api.nim +++ b/beacon_chain/rpc/rest_validator_api.nim @@ -392,21 +392,13 @@ proc installValidatorApiHandlers*(router: var RestRouter, node: BeaconNode) = qslot, proposer, qrandao, qskip_randao_verification): return RestApiResponse.jsonError(Http400, InvalidRandaoRevealValue) - let res = - case node.dag.cfg.consensusForkAtEpoch(qslot.epoch) - of ConsensusFork.Deneb: - await makeBeaconBlockForHeadAndSlot( - deneb.ExecutionPayloadForSigning, - node, qrandao, proposer, qgraffiti, qhead, qslot) - of ConsensusFork.Capella: - await makeBeaconBlockForHeadAndSlot( - capella.ExecutionPayloadForSigning, - node, qrandao, proposer, qgraffiti, qhead, qslot) - of ConsensusFork.Bellatrix: + let res = withConsensusFork( + node.dag.cfg.consensusForkAtEpoch(qslot.epoch)): + when consensusFork >= ConsensusFork.Bellatrix: await makeBeaconBlockForHeadAndSlot( - bellatrix.ExecutionPayloadForSigning, + consensusFork.ExecutionPayloadForSigning, node, qrandao, proposer, qgraffiti, qhead, qslot) - of ConsensusFork.Altair, ConsensusFork.Phase0: + else: return RestApiResponse.jsonError(Http400, InvalidSlotValueError) if res.isErr(): return RestApiResponse.jsonError(Http400, res.error()) @@ -414,7 +406,7 @@ proc installValidatorApiHandlers*(router: var RestRouter, node: BeaconNode) = return withBlck(message.blck): let data = - when forkyBlck is deneb.BeaconBlock: + when consensusFork >= ConsensusFork.Deneb: let bundle = message.blobsBundleOpt.get() let blockRoot = hash_tree_root(forkyBlck) var sidecars = newSeqOfCap[BlobSidecar](bundle.blobs.len) @@ -426,7 +418,7 @@ proc installValidatorApiHandlers*(router: var RestRouter, node: BeaconNode) = block_parent_root: forkyBlck.parent_root, proposer_index: forkyBlck.proposer_index, blob: bundle.blobs[i], - kzg_commitment: bundle.kzgs[i], + kzg_commitment: bundle.commitments[i], kzg_proof: bundle.proofs[i] ) sidecars.add(sidecar) @@ -435,18 +427,13 @@ proc installValidatorApiHandlers*(router: var RestRouter, node: BeaconNode) = `block`: forkyBlck, blob_sidecars: List[BlobSidecar, Limit MAX_BLOBS_PER_BLOCK].init(sidecars)) - elif forkyBlck is phase0.BeaconBlock or - forkyBlck is altair.BeaconBlock or - forkyBlck is bellatrix.BeaconBlock or - forkyBlck is capella.BeaconBlock: - forkyBlck else: - static: raiseAssert "produceBlockV2 received unexpected version" + forkyBlck if contentType == sszMediaType: - let headers = [("eth-consensus-version", message.blck.kind.toString())] + let headers = [("eth-consensus-version", consensusFork.toString())] RestApiResponse.sszResponse(data, headers) elif contentType == jsonMediaType: - RestApiResponse.jsonResponseWVersion(data, message.blck.kind) + RestApiResponse.jsonResponseWVersion(data, consensusFork) else: raiseAssert "preferredContentType() returns invalid content type" diff --git a/beacon_chain/spec/datatypes/deneb.nim b/beacon_chain/spec/datatypes/deneb.nim index f58a8accec..e5e2cafc3f 100644 --- a/beacon_chain/spec/datatypes/deneb.nim +++ b/beacon_chain/spec/datatypes/deneb.nim @@ -38,6 +38,7 @@ const type KzgCommitments* = List[KzgCommitment, Limit MAX_BLOB_COMMITMENTS_PER_BLOCK] + KzgProofs* = List[KzgProof, Limit MAX_BLOB_COMMITMENTS_PER_BLOCK] Blobs* = List[Blob, Limit MAX_BLOB_COMMITMENTS_PER_BLOCK] # TODO this apparently is suppposed to be SSZ-equivalent to Bytes32, but @@ -50,6 +51,12 @@ type # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.2/specs/deneb/polynomial-commitments.md#custom-types Blob* = array[BYTES_PER_FIELD_ELEMENT * FIELD_ELEMENTS_PER_BLOB, byte] + # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.4/specs/deneb/validator.md#blobsbundle + BlobsBundle* = object + commitments*: KzgCommitments + proofs*: KzgProofs + blobs*: Blobs + # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/deneb/p2p-interface.md#blobsidecar BlobSidecar* = object block_root*: Eth2Digest @@ -103,9 +110,7 @@ type ExecutionPayloadForSigning* = object executionPayload*: ExecutionPayload blockValue*: Wei - kzgs*: KzgCommitments - proofs*: seq[KZGProof] - blobs*: Blobs + blobsBundle*: BlobsBundle # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/deneb/beacon-chain.md#executionpayloadheader ExecutionPayloadHeader* = object @@ -131,12 +136,6 @@ type blob_gas_used*: uint64 # [New in Deneb:EIP4844] excess_blob_gas*: uint64 # [New in Deneb:EIP4844] - # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/deneb/validator.md#blobsbundle - BlobsBundle* = object - commitments*: seq[KZGCommitment] - proofs*: seq[KZGProof] - blobs*: seq[Blob] - ExecutePayload* = proc( execution_payload: ExecutionPayload): bool {.gcsafe, raises: [].} diff --git a/beacon_chain/spec/state_transition.nim b/beacon_chain/spec/state_transition.nim index 6b8cb9ded6..3e77163632 100644 --- a/beacon_chain/spec/state_transition.nim +++ b/beacon_chain/spec/state_transition.nim @@ -380,7 +380,7 @@ func partialBeaconBlock*( # https://github.com/ethereum/consensus-specs/blob/v1.3.0/specs/deneb/validator.md#constructing-the-beaconblockbody when consensusFork >= ConsensusFork.Deneb: - res.body.blob_kzg_commitments = execution_payload.kzgs + res.body.blob_kzg_commitments = execution_payload.blobsBundle.commitments res diff --git a/beacon_chain/sync/request_manager.nim b/beacon_chain/sync/request_manager.nim index 8fe3b3ff57..d37a13d504 100644 --- a/beacon_chain/sync/request_manager.nim +++ b/beacon_chain/sync/request_manager.nim @@ -292,7 +292,7 @@ proc getMissingBlobs(rman: RequestManager): seq[BlobIdentifier] = warn "quarantine missing blobs, but missing indices is empty", blk=blobless.root, indices=rman.blobQuarantine[].blobIndices(blobless.root), - kzgs=len(blobless.message.body.blob_kzg_commitments) + commitments=len(blobless.message.body.blob_kzg_commitments) for idx in missing.indices: let id = BlobIdentifier(block_root: blobless.root, index: idx) if id notin fetches: @@ -302,7 +302,7 @@ proc getMissingBlobs(rman: RequestManager): seq[BlobIdentifier] = warn "missing blob handler found blobless block with all blobs", blk=blobless.root, indices=rman.blobQuarantine[].blobIndices(blobless.root), - kzgs=len(blobless.message.body.blob_kzg_commitments) + commitments=len(blobless.message.body.blob_kzg_commitments) discard rman.blockVerifier(ForkedSignedBeaconBlock.init(blobless), false) rman.quarantine[].removeBlobless(blobless) @@ -356,4 +356,3 @@ proc stop*(rman: RequestManager) = rman.blockLoopFuture.cancelSoon() if not(isNil(rman.blobLoopFuture)): rman.blobLoopFuture.cancelSoon() - diff --git a/beacon_chain/validators/beacon_validators.nim b/beacon_chain/validators/beacon_validators.nim index 7face4f0d2..ee27a0027c 100644 --- a/beacon_chain/validators/beacon_validators.nim +++ b/beacon_chain/validators/beacon_validators.nim @@ -88,9 +88,6 @@ declarePublicGauge(attached_validator_balance_total, logScope: topics = "beacval" type - BlobsBundle = tuple[blobs: deneb.Blobs, - kzgs: KzgCommitments, - proofs: seq[kzg_abi.KZGProof]] ForkedBlockResult = Result[tuple[blck: ForkedBeaconBlock, blockValue: Wei, @@ -541,10 +538,7 @@ proc makeBeaconBlockForHeadAndSlot*( var blobsBundleOpt = Opt.none(BlobsBundle) when payload is deneb.ExecutionPayloadForSigning: - let bb: BlobsBundle = (blobs: payload.blobs, - kzgs: payload.kzgs, - proofs: payload.proofs) - blobsBundleOpt = Opt.some(bb) + blobsBundleOpt = Opt.some(payload.blobsBundle) return if blck.isOk: ok((blck.get, payload.blockValue, blobsBundleOpt)) else: @@ -1176,10 +1170,11 @@ proc proposeBlockAux( .registerBlock(validator_index, validator.pubkey, slot, signingRoot) let blobSidecarsOpt = - when forkyBlck is deneb.BeaconBlock: + when consensusFork >= ConsensusFork.Deneb: var sidecars: seq[BlobSidecar] let bundle = collectedBids.engineBlockFut.read.get().blobsBundleOpt.get - let (blobs, kzgs, proofs) = (bundle.blobs, bundle.kzgs, bundle.proofs) + let (blobs, commitments, proofs) = ( + bundle.blobs, bundle.commitments, bundle.proofs) for i in 0.. Date: Sat, 4 Nov 2023 15:20:34 +0100 Subject: [PATCH 25/35] introduce `BlobId` type for tracking blob subnets (#5560) Instead of mixing up `SubnetId` (attestation subnet type) for blobs, introduce dedicated `BlobId` type. --- beacon_chain/gossip_processing/eth2_processor.nim | 4 ++-- beacon_chain/gossip_processing/gossip_validation.nim | 6 +++--- beacon_chain/networking/eth2_network.nim | 2 +- beacon_chain/nimbus_beacon_node.nim | 10 +++++----- beacon_chain/spec/datatypes/base.nim | 8 ++++++++ beacon_chain/spec/network.nim | 12 ++++++------ tests/test_honest_validator.nim | 2 +- 7 files changed, 26 insertions(+), 18 deletions(-) diff --git a/beacon_chain/gossip_processing/eth2_processor.nim b/beacon_chain/gossip_processing/eth2_processor.nim index f26f099d5c..71f22f66d0 100644 --- a/beacon_chain/gossip_processing/eth2_processor.nim +++ b/beacon_chain/gossip_processing/eth2_processor.nim @@ -275,7 +275,7 @@ proc processSignedBeaconBlock*( proc processSignedBlobSidecar*( self: var Eth2Processor, src: MsgSource, - signedBlobSidecar: deneb.SignedBlobSidecar, idx: BlobIndex): ValidationRes = + signedBlobSidecar: deneb.SignedBlobSidecar, subnet_id: BlobId): ValidationRes = let wallTime = self.getCurrentBeaconTime() (afterGenesis, wallSlot) = wallTime.toSlot() @@ -296,7 +296,7 @@ proc processSignedBlobSidecar*( let v = self.dag.validateBlobSidecar(self.quarantine, self.blobQuarantine, - signedBlobSidecar, wallTime, idx) + signedBlobSidecar, wallTime, subnet_id) if v.isErr(): debug "Dropping blob", error = v.error() diff --git a/beacon_chain/gossip_processing/gossip_validation.nim b/beacon_chain/gossip_processing/gossip_validation.nim index 3d58cf22eb..8ec4538f0e 100644 --- a/beacon_chain/gossip_processing/gossip_validation.nim +++ b/beacon_chain/gossip_processing/gossip_validation.nim @@ -305,12 +305,12 @@ template validateBeaconBlockBellatrix( # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.2/specs/deneb/p2p-interface.md#blob_sidecar_subnet_id proc validateBlobSidecar*( dag: ChainDAGRef, quarantine: ref Quarantine, - blobQuarantine: ref BlobQuarantine,sbs: SignedBlobSidecar, - wallTime: BeaconTime, idx: BlobIndex): Result[void, ValidationError] = + blobQuarantine: ref BlobQuarantine, sbs: SignedBlobSidecar, + wallTime: BeaconTime, subnet_id: BlobId): Result[void, ValidationError] = # [REJECT] The sidecar is for the correct topic -- # i.e. sidecar.index matches the topic {index}. - if sbs.message.index != idx: + if sbs.message.index != subnet_id: return dag.checkedReject("SignedBlobSidecar: mismatched gossip topic index") if dag.getBlockRef(sbs.message.block_root).isSome(): diff --git a/beacon_chain/networking/eth2_network.nim b/beacon_chain/networking/eth2_network.nim index d572ba005d..23f6c5ad82 100644 --- a/beacon_chain/networking/eth2_network.nim +++ b/beacon_chain/networking/eth2_network.nim @@ -2678,7 +2678,7 @@ proc broadcastBeaconBlock*( node.broadcast(topic, blck) proc broadcastBlobSidecar*( - node: Eth2Node, subnet_id: SubnetId, blob: deneb.SignedBlobSidecar): + node: Eth2Node, subnet_id: BlobId, blob: deneb.SignedBlobSidecar): Future[SendResult] = let forkPrefix = node.forkDigestAtEpoch(node.getWallEpoch) diff --git a/beacon_chain/nimbus_beacon_node.nim b/beacon_chain/nimbus_beacon_node.nim index 1721505eca..39c91cd110 100644 --- a/beacon_chain/nimbus_beacon_node.nim +++ b/beacon_chain/nimbus_beacon_node.nim @@ -1733,18 +1733,18 @@ proc installMessageValidators(node: BeaconNode) = MsgSource.gossip, msg))) when consensusFork >= ConsensusFork.Deneb: - # blob_sidecar_{index} + # blob_sidecar_{subnet_id} # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.2/specs/deneb/p2p-interface.md#blob_sidecar_subnet_id - for i in 0 ..< BLOB_SIDECAR_SUBNET_COUNT: + for it in BlobId: closureScope: # Needed for inner `proc`; don't lift it out of loop. - let idx = i + let subnet_id = it node.network.addValidator( - getBlobSidecarTopic(digest, SubnetId(idx)), proc ( + getBlobSidecarTopic(digest, subnet_id), proc ( signedBlobSidecar: SignedBlobSidecar ): ValidationResult = toValidationResult( node.processor[].processSignedBlobSidecar( - MsgSource.gossip, signedBlobSidecar, idx))) + MsgSource.gossip, signedBlobSidecar, subnet_id))) node.installLightClientMessageValidators() diff --git a/beacon_chain/spec/datatypes/base.nim b/beacon_chain/spec/datatypes/base.nim index 0d2997441a..6b311a9b59 100644 --- a/beacon_chain/spec/datatypes/base.nim +++ b/beacon_chain/spec/datatypes/base.nim @@ -187,6 +187,13 @@ type ## The `SubnetId` type is constrained to values in the range ## `[0, ATTESTATION_SUBNET_COUNT)` during initialization. + BlobId* = distinct uint8 + ## The blob id maps which gossip subscription to use to publish a + ## blob sidecar - it is distinct from the CommitteeIndex in particular + ## + ## The `BlobId` type is constrained to values in the range + ## `[0, BLOB_SIDECAR_SUBNET_COUNT)` during initialization. + # BitVector[4] in the spec, ie 4 bits which end up encoded as a byte for # SSZ / hashing purposes JustificationBits* = distinct uint8 @@ -603,6 +610,7 @@ template makeLimitedU64*(T: untyped, limit: uint64) = makeLimitedU64(CommitteeIndex, MAX_COMMITTEES_PER_SLOT) makeLimitedU64(SubnetId, ATTESTATION_SUBNET_COUNT) +makeLimitedU64(BlobId, BLOB_SIDECAR_SUBNET_COUNT) const validatorIndexLimit = min(uint64(int32.high), VALIDATOR_REGISTRY_LIMIT) diff --git a/beacon_chain/spec/network.nim b/beacon_chain/spec/network.nim index 7f86eea418..c03b8d0e47 100644 --- a/beacon_chain/spec/network.nim +++ b/beacon_chain/spec/network.nim @@ -100,14 +100,14 @@ func getSyncCommitteeContributionAndProofTopic*(forkDigest: ForkDigest): string ## For subscribing and unsubscribing to/from a subnet. eth2Prefix(forkDigest) & "sync_committee_contribution_and_proof/ssz_snappy" -# https://github.com/ethereum/consensus-specs/blob/v1.3.0/specs/deneb/p2p-interface.md#blob_sidecar_index +# https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.4/specs/deneb/p2p-interface.md#blob_sidecar_subnet_id func getBlobSidecarTopic*(forkDigest: ForkDigest, - subnet_id: SubnetId): string = + subnet_id: BlobId): string = eth2Prefix(forkDigest) & "blob_sidecar_" & $subnet_id & "/ssz_snappy" # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/deneb/validator.md#sidecar -func compute_subnet_for_blob_sidecar*(blob_index: BlobIndex): SubnetId = - SubnetId(blob_index mod BLOB_SIDECAR_SUBNET_COUNT) +func compute_subnet_for_blob_sidecar*(blob_index: BlobIndex): BlobId = + BlobId(blob_index mod BLOB_SIDECAR_SUBNET_COUNT) # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/altair/light-client/p2p-interface.md#light_client_finality_update func getLightClientFinalityUpdateTopic*(forkDigest: ForkDigest): string = @@ -220,5 +220,5 @@ func getSyncSubnets*( res iterator blobSidecarTopics*(forkDigest: ForkDigest): string = - for i in 0.SubnetId ..< static(BLOB_SIDECAR_SUBNET_COUNT.SubnetId): - yield getBlobSidecarTopic(forkDigest, i) + for subnet_id in BlobId: + yield getBlobSidecarTopic(forkDigest, subnet_id) diff --git a/tests/test_honest_validator.nim b/tests/test_honest_validator.nim index 509ffe7760..969a67d808 100644 --- a/tests/test_honest_validator.nim +++ b/tests/test_honest_validator.nim @@ -77,7 +77,7 @@ suite "Honest validator": "/eth2/00000000/sync_committee_1/ssz_snappy" getSyncCommitteeTopic(forkDigest, SyncSubcommitteeIndex(3)) == "/eth2/00000000/sync_committee_3/ssz_snappy" - getBlobSidecarTopic(forkDigest, SubnetId(1)) == + getBlobSidecarTopic(forkDigest, BlobId(1)) == "/eth2/00000000/blob_sidecar_1/ssz_snappy" toSeq(blobSidecarTopics(forkDigest)) == ["/eth2/00000000/blob_sidecar_0/ssz_snappy", From 6958f67af3f477b5dc106aa2f267afb047c26205 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Sat, 4 Nov 2023 20:35:41 +0100 Subject: [PATCH 26/35] add test runner for v1.4.0-beta.4 Merkle proof tests (#5567) Create a test runner for validating the new `Merkle proof` Deneb tests that are added with `v1.4.0-beta.4` specs. --- .../consensus_spec_tests_preset.nim | 1 + .../test_fixture_merkle_proof.nim | 74 +++++++++++++++++++ 2 files changed, 75 insertions(+) create mode 100644 tests/consensus_spec/test_fixture_merkle_proof.nim diff --git a/tests/consensus_spec/consensus_spec_tests_preset.nim b/tests/consensus_spec/consensus_spec_tests_preset.nim index 24bd8bf01e..cc3e3785da 100644 --- a/tests/consensus_spec/consensus_spec_tests_preset.nim +++ b/tests/consensus_spec/consensus_spec_tests_preset.nim @@ -20,6 +20,7 @@ import ./test_fixture_light_client_single_merkle_proof, ./test_fixture_light_client_sync, ./test_fixture_light_client_update_ranking, + ./test_fixture_merkle_proof, ./test_fixture_sanity_blocks, ./test_fixture_sanity_slots, ./test_fixture_transition diff --git a/tests/consensus_spec/test_fixture_merkle_proof.nim b/tests/consensus_spec/test_fixture_merkle_proof.nim new file mode 100644 index 0000000000..828a4e048a --- /dev/null +++ b/tests/consensus_spec/test_fixture_merkle_proof.nim @@ -0,0 +1,74 @@ +# beacon_chain +# Copyright (c) 2023 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +{.used.} + +import + # Standard library + std/[sequtils, streams], + # Status libraries + stew/bitops2, + # Third-party + yaml, + # Beacon chain internals + ../../beacon_chain/spec/helpers, + # Test utilities + ../testutil, + ./fixtures_utils, ./os_ops + +proc runTest[T](suiteName, path: string, objType: typedesc[T]) = + test "Merkle proof - Single merkle proof - " & path.relativePath(SszTestsDir): + type + TestProof = object + leaf: string + leaf_index: GeneralizedIndex + branch: seq[string] + + let + proof = block: + let s = openFileStream(path/"proof.yaml") + defer: close(s) + var res: TestProof + yaml.load(s, res) + res + + obj = newClone(parseTest(path/"object.ssz_snappy", SSZ, T)) + + var computedProof = newSeq[Eth2Digest](log2trunc(proof.leaf_index)) + build_proof(obj[], proof.leaf_index, computedProof).get + + check: + computedProof == proof.branch.mapIt(Eth2Digest.fromHex(it)) + is_valid_merkle_branch( + Eth2Digest.fromHex(proof.leaf), + computedProof, + log2trunc(proof.leaf_index), + get_subtree_index(proof.leaf_index), + hash_tree_root(obj[])) + +suite "EF - Merkle proof" & preset(): + const presetPath = SszTestsDir/const_preset + for kind, path in walkDir(presetPath, relative = true, checkDir = true): + let testsPath = presetPath/path/"merkle_proof"/"single_merkle_proof" + if kind != pcDir or not dirExists(testsPath): + continue + let fork = forkForPathComponent(path).valueOr: + test "Merkle proof - Single merkle proof - " & path: + skip() + continue + for kind, path in walkDir(testsPath, relative = true, checkDir = true): + let suitePath = testsPath/path + if kind != pcDir or not dirExists(suitePath): + continue + let objName = path + withConsensusFork(fork): + for kind, path in walkDir(suitePath, relative = true, checkDir = true): + case objName + of "BeaconBlockBody": + runTest(suiteName, suitePath/path, consensusFork.BeaconBlockBody) + else: + raiseAssert "Unknown test object: " & suitePath/path From 8d46809a5c8488ef13728beb5853d9d780d00c77 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Sat, 4 Nov 2023 20:36:01 +0100 Subject: [PATCH 27/35] skip upcoming FC tests for intentional block reorgs until implemented (#5566) v1.4.0-beta.4 adds tests for intentional block reorgs. To reflect the implementation status, skip those tests for now and mark them as such. --- tests/consensus_spec/test_fixture_fork_choice.nim | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/consensus_spec/test_fixture_fork_choice.nim b/tests/consensus_spec/test_fixture_fork_choice.nim index c831652796..f4edfc4c40 100644 --- a/tests/consensus_spec/test_fixture_fork_choice.nim +++ b/tests/consensus_spec/test_fixture_fork_choice.nim @@ -354,6 +354,12 @@ proc runTest(suiteName: static[string], path: string, fork: ConsensusFork) = "too_late_for_merge", "block_lookup_failed", "all_valid", + + # TODO intentional reorgs + "should_override_forkchoice_update__false", + "should_override_forkchoice_update__true", + "basic_is_parent_root", + "basic_is_head_root", ] test suiteName & " - " & path.relativePath(SszTestsDir): From f14389bb8440f437e9cea6f2ba6d6386fbdd2219 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Sat, 4 Nov 2023 20:36:12 +0100 Subject: [PATCH 28/35] avoid perpetually sending blobs to peers (#5563) Fix regression from #4808 where blobs that are already known are issued ACCEPT verdict, propagating them to peers over and over again. `validateBlobSidecar` contains the correct IGNORE logic. Moved it above the expensive checks to retain the performance of the check. --- beacon_chain/gossip_processing/eth2_processor.nim | 7 +------ beacon_chain/gossip_processing/gossip_validation.nim | 12 ++++++------ 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/beacon_chain/gossip_processing/eth2_processor.nim b/beacon_chain/gossip_processing/eth2_processor.nim index 71f22f66d0..87406275b5 100644 --- a/beacon_chain/gossip_processing/eth2_processor.nim +++ b/beacon_chain/gossip_processing/eth2_processor.nim @@ -287,12 +287,7 @@ proc processSignedBlobSidecar*( # Potential under/overflows are fine; would just create odd metrics and logs let delay = wallTime - signedBlobSidecar.message.slot.start_beacon_time - - if self.blobQuarantine[].hasBlob(signedBlobSidecar.message): - debug "Blob received, already in quarantine", delay - return ValidationRes.ok - else: - debug "Blob received", delay + debug "Blob received", delay let v = self.dag.validateBlobSidecar(self.quarantine, self.blobQuarantine, diff --git a/beacon_chain/gossip_processing/gossip_validation.nim b/beacon_chain/gossip_processing/gossip_validation.nim index 8ec4538f0e..5874a8349a 100644 --- a/beacon_chain/gossip_processing/gossip_validation.nim +++ b/beacon_chain/gossip_processing/gossip_validation.nim @@ -331,6 +331,12 @@ proc validateBlobSidecar*( if not (sbs.message.slot > dag.finalizedHead.slot): return errIgnore("SignedBlobSidecar: slot already finalized") + # [IGNORE] The sidecar is the only sidecar with valid signature + # received for the tuple (sidecar.block_root, sidecar.index). + if blobQuarantine[].hasBlob(sbs.message): + return errIgnore( + "SignedBlobSidecar: already have blob with valid signature") + # [IGNORE] The block's parent (defined by block.parent_root) has # been seen (via both gossip and non-gossip sources) (a client MAY # queue blocks for processing once the parent block is retrieved). @@ -376,12 +382,6 @@ proc validateBlobSidecar*( sbs.signature): return dag.checkedReject("SignedBlobSidecar: invalid blob signature") - # [IGNORE] The sidecar is the only sidecar with valid signature - # received for the tuple (sidecar.block_root, sidecar.index). - if blobQuarantine[].hasBlob(sbs.message): - return errIgnore( - "SignedBlobSidecar: already have blob with valid signature") - ok() From 28b84ff93b412cdec40d9edb7d79e818e0491b0a Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Sun, 5 Nov 2023 09:13:57 +0100 Subject: [PATCH 29/35] extend `hasBlob` to all blocks from same proposer and slot (#5568) `v1.4.0-beta.4` made the Gossip rules more strict and now requires to ignore blobs from other branches if there are equivocating blocks. Those blobs are only requestable via Req/Resp. --- .../blob_quarantine.nim | 17 +++++++++--- .../gossip_processing/gossip_validation.nim | 26 +++++++++++-------- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/beacon_chain/consensus_object_pools/blob_quarantine.nim b/beacon_chain/consensus_object_pools/blob_quarantine.nim index c7a78b4048..cd62e1535e 100644 --- a/beacon_chain/consensus_object_pools/blob_quarantine.nim +++ b/beacon_chain/consensus_object_pools/blob_quarantine.nim @@ -37,8 +37,8 @@ func put*(quarantine: var BlobQuarantine, blobSidecar: ref BlobSidecar) = oldest_blob_key = k break quarantine.blobs.del oldest_blob_key - discard quarantine.blobs.hasKeyOrPut((blobSidecar.block_root, - blobSidecar.index), blobSidecar) + discard quarantine.blobs.hasKeyOrPut( + (blobSidecar.block_root, blobSidecar.index), blobSidecar) func blobIndices*(quarantine: BlobQuarantine, digest: Eth2Digest): seq[BlobIndex] = @@ -48,8 +48,17 @@ func blobIndices*(quarantine: BlobQuarantine, digest: Eth2Digest): r.add(i) r -func hasBlob*(quarantine: BlobQuarantine, blobSidecar: BlobSidecar): bool = - quarantine.blobs.hasKey((blobSidecar.block_root, blobSidecar.index)) +func hasBlob*( + quarantine: BlobQuarantine, + slot: Slot, + proposer_index: uint64, + index: BlobIndex): bool = + for blob_sidecar in quarantine.blobs.values: + if blob_sidecar.slot == slot and + blob_sidecar.proposer_index == proposer_index and + blob_sidecar.index == index: + return true + false func popBlobs*(quarantine: var BlobQuarantine, digest: Eth2Digest): seq[ref BlobSidecar] = diff --git a/beacon_chain/gossip_processing/gossip_validation.nim b/beacon_chain/gossip_processing/gossip_validation.nim index 5874a8349a..dbac3b26d2 100644 --- a/beacon_chain/gossip_processing/gossip_validation.nim +++ b/beacon_chain/gossip_processing/gossip_validation.nim @@ -307,15 +307,16 @@ proc validateBlobSidecar*( dag: ChainDAGRef, quarantine: ref Quarantine, blobQuarantine: ref BlobQuarantine, sbs: SignedBlobSidecar, wallTime: BeaconTime, subnet_id: BlobId): Result[void, ValidationError] = + # Some of the checks below have been reordered compared to the spec, to + # perform the cheap checks first - in particular, we want to avoid loading + # an `EpochRef` and checking signatures. This reordering might lead to + # different IGNORE/REJECT results in turn affecting gossip scores. # [REJECT] The sidecar is for the correct topic -- # i.e. sidecar.index matches the topic {index}. if sbs.message.index != subnet_id: return dag.checkedReject("SignedBlobSidecar: mismatched gossip topic index") - if dag.getBlockRef(sbs.message.block_root).isSome(): - return errIgnore("SignedBlobSidecar: already have block") - # [IGNORE] The sidecar is not from a future slot (with a # MAXIMUM_GOSSIP_CLOCK_DISPARITY allowance) -- i.e. validate that # sidecar.slot <= current_slot (a client MAY queue future sidecars @@ -331,11 +332,14 @@ proc validateBlobSidecar*( if not (sbs.message.slot > dag.finalizedHead.slot): return errIgnore("SignedBlobSidecar: slot already finalized") - # [IGNORE] The sidecar is the only sidecar with valid signature - # received for the tuple (sidecar.block_root, sidecar.index). - if blobQuarantine[].hasBlob(sbs.message): - return errIgnore( - "SignedBlobSidecar: already have blob with valid signature") + # [IGNORE] The sidecar is the first sidecar for the tuple + # (block_header.slot, block_header.proposer_index, blob_sidecar.index) + # with valid header signature, sidecar inclusion proof, and kzg proof. + if dag.getBlockRef(sbs.message.block_root).isSome(): + return errIgnore("SignedBlobSidecar: already have block") + if blobQuarantine[].hasBlob( + sbs.message.slot, sbs.message.proposer_index, sbs.message.index): + return errIgnore("SignedBlobSidecar: already have valid blob from same proposer") # [IGNORE] The block's parent (defined by block.parent_root) has # been seen (via both gossip and non-gossip sources) (a client MAY @@ -498,7 +502,7 @@ proc validateBeaconBlock*( blockRoot = shortLog(signed_beacon_block.root), blck = shortLog(signed_beacon_block.message), signature = shortLog(signed_beacon_block.signature) - return errIgnore("BeaconBlock: Parent not found") + return errIgnore("BeaconBlock: parent not found") # Continues block parent validity checking in optimistic case, where it does # appear as a `BlockRef` (and not handled above) but isn't usable for gossip @@ -539,12 +543,12 @@ proc validateBeaconBlock*( let proposer = getProposer( dag, parent, signed_beacon_block.message.slot).valueOr: - warn "cannot compute proposer for message" + warn "cannot compute proposer for block" return errIgnore("BeaconBlock: Cannot compute proposer") # internal issue if uint64(proposer) != signed_beacon_block.message.proposer_index: quarantine[].addUnviable(signed_beacon_block.root) - return dag.checkedReject("BeaconBlock: Unexpected proposer proposer") + return dag.checkedReject("BeaconBlock: Unexpected proposer") # [REJECT] The proposer signature, signed_beacon_block.signature, is valid # with respect to the proposer_index pubkey. From e4dacc30c4c347399e7951c602807caa9a916e05 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Sun, 5 Nov 2023 09:14:47 +0100 Subject: [PATCH 30/35] fix validation of blob index against gossip topic (#5569) In `v1.4.0-alpha.0`, the blob index validation on gossip was changed to use `compute_subnet_for_blob_sidecar` instead of having a separate topic for each individual blob. We updated the spec reference in #5363 without updating the code accordingly. Fixing this now, and also adding the new `MAX_BLOBS_PER_BLOCK` check from `v1.4.0-beta.3` as it shares the theme. --- .../gossip_processing/gossip_validation.nim | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/beacon_chain/gossip_processing/gossip_validation.nim b/beacon_chain/gossip_processing/gossip_validation.nim index dbac3b26d2..49248b832c 100644 --- a/beacon_chain/gossip_processing/gossip_validation.nim +++ b/beacon_chain/gossip_processing/gossip_validation.nim @@ -312,15 +312,19 @@ proc validateBlobSidecar*( # an `EpochRef` and checking signatures. This reordering might lead to # different IGNORE/REJECT results in turn affecting gossip scores. - # [REJECT] The sidecar is for the correct topic -- - # i.e. sidecar.index matches the topic {index}. - if sbs.message.index != subnet_id: - return dag.checkedReject("SignedBlobSidecar: mismatched gossip topic index") + # [REJECT] The sidecar's index is consistent with `MAX_BLOBS_PER_BLOCK` + # -- i.e. `blob_sidecar.index < MAX_BLOBS_PER_BLOCK` + if not (sbs.message.index < MAX_BLOBS_PER_BLOCK): + return dag.checkedReject("SignedBlobSidecar: index inconsistent") + + # [REJECT] The sidecar is for the correct subnet -- i.e. + # `compute_subnet_for_blob_sidecar(blob_sidecar.index) == subnet_id`. + if not (compute_subnet_for_blob_sidecar(sbs.message.index) == subnet_id): + return dag.checkedReject("SignedBlobSidecar: subnet incorrect") # [IGNORE] The sidecar is not from a future slot (with a - # MAXIMUM_GOSSIP_CLOCK_DISPARITY allowance) -- i.e. validate that - # sidecar.slot <= current_slot (a client MAY queue future sidecars - # for processing at the appropriate slot). + # `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. validate that + # `block_header.slot <= current_slot` (a client MAY queue future sidecars if not (sbs.message.slot <= (wallTime + MAXIMUM_GOSSIP_CLOCK_DISPARITY).slotOrZero): return errIgnore("SignedBlobSidecar: slot too high") From 40196b75d0c6bcba665b75bf4509e8edec6ecd4e Mon Sep 17 00:00:00 2001 From: xiaolou86 <20718693+xiaolou86@users.noreply.github.com> Date: Mon, 6 Nov 2023 11:56:07 +0800 Subject: [PATCH 31/35] docs: fix typos (#5571) --- beacon_chain/sync/README.md | 2 +- docs/e2store.md | 2 +- docs/the_nimbus_book/src/web3signer.md | 2 +- ncli/README.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/beacon_chain/sync/README.md b/beacon_chain/sync/README.md index 162aa4eb1a..282615f1c8 100644 --- a/beacon_chain/sync/README.md +++ b/beacon_chain/sync/README.md @@ -21,7 +21,7 @@ Blocks are received by batch: - in case of failure: - `push(SyncQueue, SyncRequest)` is called to reschedule the sync request. -Every second when sync is not in progress, the beacon node will ask the RequestManager to download all missing blocks currently in quarantaine. +Every second when sync is not in progress, the beacon node will ask the RequestManager to download all missing blocks currently in quarantine. - via `handleMissingBlocks` - which calls `fetchAncestorBlocks` - which asynchronously enqueue the request in the SharedBlockQueue `AsyncQueue[BlockEntry]`. diff --git a/docs/e2store.md b/docs/e2store.md index 725fda5035..16712b5a19 100644 --- a/docs/e2store.md +++ b/docs/e2store.md @@ -327,7 +327,7 @@ An era is typically `8192` slots (in the mainnet configuration), or roughly 27.3 ## What happens after the merge? -Era files will store execution block contents, but not execution states (these are too large) - a full era history thus gives the full ethereum history from the merge onwards for convenient cold storage. Work is underway to similarily cover the rest of history. +Era files will store execution block contents, but not execution states (these are too large) - a full era history thus gives the full ethereum history from the merge onwards for convenient cold storage. Work is underway to similarly cover the rest of history. ## Which state should be stored in the era file? diff --git a/docs/the_nimbus_book/src/web3signer.md b/docs/the_nimbus_book/src/web3signer.md index e5f7734616..c8a7f0b12d 100644 --- a/docs/the_nimbus_book/src/web3signer.md +++ b/docs/the_nimbus_book/src/web3signer.md @@ -40,7 +40,7 @@ The fields have the following semantics: 5. `remote` - An URL of a remote signing server. 6. `remotes` - A [distributed keystore](#distributed-keystores) configuration including two or more remote signing servers. 7. `ignore_ssl_verification` - An optional boolean flag allowing the use of self-signed certificates by the signing server. -8. `proven_block_properties` - When the `verifying-web3signer` type is used, this is a list of locations withing the SSZ block body for which the block signing requests will contain additional Merkle proofs, allowing the signer to verify certain details about the signed blocks (e.g. the `fee_recipient` value). +8. `proven_block_properties` - When the `verifying-web3signer` type is used, this is a list of locations within the SSZ block body for which the block signing requests will contain additional Merkle proofs, allowing the signer to verify certain details about the signed blocks (e.g. the `fee_recipient` value). !!! info The current version of the remote keystore format is `3` which adds support for the experimental [verifying web3signer setups](#verifying-web3signer). diff --git a/ncli/README.md b/ncli/README.md index 72aae6559d..f645007ed9 100644 --- a/ncli/README.md +++ b/ncli/README.md @@ -1,6 +1,6 @@ # Introduction -`ncli` is a set of low level / debugging tools to interact with the nimbus [beacon chain specification](https://github.com/ethereum/consensus-specs/tree/dev/specs) implementation, simliar to [zcli](https://github.com/protolambda/zcli). With it, you explore SSZ, make state transitions and compute hash tree roots. +`ncli` is a set of low level / debugging tools to interact with the nimbus [beacon chain specification](https://github.com/ethereum/consensus-specs/tree/dev/specs) implementation, similar to [zcli](https://github.com/protolambda/zcli). With it, you explore SSZ, make state transitions and compute hash tree roots. # Tools From d8a7f0df81761b0861ed11f6b1a5d0aa58d17ae9 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Mon, 6 Nov 2023 07:48:43 +0100 Subject: [PATCH 32/35] update Deneb for blob sidecar inclusion proofs (#5565) `BlobSidecar` is no longer signed, instead use Merkle proof to link blobs with block. - https://github.com/ethereum/consensus-specs/pull/3531 Associated beacon-API / builder-specs still TBD; minimal changes done to compile in similar style to previous spec, but not standardized yet. - https://github.com/ethereum/beacon-APIs/pull/369 - https://github.com/ethereum/builder-specs/pull/90 --- AllTests-mainnet.md | 5 +- ConsensusSpecPreset-mainnet.md | 23 ++- ConsensusSpecPreset-minimal.md | 26 ++- beacon_chain/beacon_chain_db.nim | 3 +- .../blob_quarantine.nim | 10 +- .../gossip_processing/eth2_processor.nim | 25 ++- .../gossip_processing/gossip_validation.nim | 163 ++++++++++++------ beacon_chain/networking/eth2_network.nim | 2 +- beacon_chain/nimbus_beacon_node.nim | 8 +- beacon_chain/rpc/rest_beacon_api.nim | 42 ++--- beacon_chain/rpc/rest_validator_api.nim | 21 +-- beacon_chain/spec/datatypes/base.nim | 2 +- beacon_chain/spec/datatypes/constants.nim | 3 - beacon_chain/spec/datatypes/deneb.nim | 73 ++++---- .../eth2_apis/eth2_rest_serialization.nim | 43 +++-- beacon_chain/spec/eth2_apis/rest_types.nim | 6 +- beacon_chain/spec/forks.nim | 17 +- beacon_chain/spec/helpers.nim | 48 ++++++ beacon_chain/spec/mev/deneb_mev.nim | 15 +- beacon_chain/spec/signatures.nim | 30 ---- beacon_chain/sync/request_manager.nim | 8 +- beacon_chain/sync/sync_manager.nim | 9 +- beacon_chain/sync/sync_queue.nim | 4 +- .../validator_client/block_service.nim | 44 ++--- beacon_chain/validators/beacon_validators.nim | 107 ++---------- beacon_chain/validators/message_router.nim | 30 ++-- .../validators/message_router_mev.nim | 5 +- beacon_chain/validators/validator_pool.nim | 15 -- .../test_fixture_ssz_consensus_objects.nim | 1 - tests/test_beacon_chain_db.nim | 20 ++- tests/test_message_signatures.nim | 30 ---- tests/test_rest_json_serialization.nim | 7 +- tests/test_signing_node.nim | 2 +- tests/test_sync_manager.nim | 12 +- vendor/nim-eth2-scenarios | 2 +- 35 files changed, 433 insertions(+), 428 deletions(-) diff --git a/AllTests-mainnet.md b/AllTests-mainnet.md index 007d509785..a67913aeb3 100644 --- a/AllTests-mainnet.md +++ b/AllTests-mainnet.md @@ -358,7 +358,6 @@ OK: 4/4 Fail: 0/4 Skip: 0/4 ```diff + Aggregate and proof signatures OK + Attestation signatures OK -+ Blob sidecar signatures OK + Deposit signatures OK + Slot signatures OK + Sync committee message signatures OK @@ -366,7 +365,7 @@ OK: 4/4 Fail: 0/4 Skip: 0/4 + Sync committee signed contribution and proof signatures OK + Voluntary exit signatures OK ``` -OK: 9/9 Fail: 0/9 Skip: 0/9 +OK: 8/8 Fail: 0/8 Skip: 0/8 ## Network metadata ```diff + goerli OK @@ -715,4 +714,4 @@ OK: 2/2 Fail: 0/2 Skip: 0/2 OK: 9/9 Fail: 0/9 Skip: 0/9 ---TOTAL--- -OK: 404/409 Fail: 0/409 Skip: 5/409 +OK: 403/408 Fail: 0/408 Skip: 5/408 diff --git a/ConsensusSpecPreset-mainnet.md b/ConsensusSpecPreset-mainnet.md index a246091457..d88b377a8b 100644 --- a/ConsensusSpecPreset-mainnet.md +++ b/ConsensusSpecPreset-mainnet.md @@ -2258,7 +2258,6 @@ OK: 34/34 Fail: 0/34 Skip: 0/34 + Testing SignedBLSToExecutionChange OK + Testing SignedBeaconBlock OK + Testing SignedBeaconBlockHeader OK -+ Testing SignedBlobSidecar OK + Testing SignedContributionAndProof OK + Testing SignedVoluntaryExit OK + Testing SigningData OK @@ -2271,7 +2270,7 @@ OK: 34/34 Fail: 0/34 Skip: 0/34 + Testing VoluntaryExit OK + Testing Withdrawal OK ``` -OK: 49/49 Fail: 0/49 Skip: 0/49 +OK: 48/48 Fail: 0/48 Skip: 0/48 ## EF - Deneb - Sanity - Blocks [Preset: mainnet] ```diff + [Invalid] EF - Deneb - Sanity - Blocks - invalid_all_zeroed_sig [Preset: mainnet] OK @@ -2411,6 +2410,11 @@ OK: 4/4 Fail: 0/4 Skip: 0/4 + Light client - Single merkle proof - mainnet/deneb/light_client/single_merkle_proof/Beacon OK ``` OK: 14/14 Fail: 0/14 Skip: 0/14 +## EF - Merkle proof [Preset: mainnet] +```diff ++ Merkle proof - Single merkle proof - mainnet/deneb/merkle_proof/single_merkle_proof/Beacon OK +``` +OK: 1/1 Fail: 0/1 Skip: 0/1 ## EF - Phase 0 - Epoch Processing - Effective balance updates [Preset: mainnet] ```diff + Effective balance updates - effective_balance_hysteresis [Preset: mainnet] OK @@ -2825,6 +2829,8 @@ OK: 40/40 Fail: 0/40 Skip: 0/40 + ForkChoice - mainnet/altair/fork_choice/get_head/pyspec_tests/proposer_boost_correct_head OK + ForkChoice - mainnet/altair/fork_choice/get_head/pyspec_tests/shorter_chain_but_heavier_we OK + ForkChoice - mainnet/altair/fork_choice/get_head/pyspec_tests/split_tie_breaker_no_attesta OK + ForkChoice - mainnet/altair/fork_choice/get_proposer_head/pyspec_tests/basic_is_head_root Skip + ForkChoice - mainnet/altair/fork_choice/get_proposer_head/pyspec_tests/basic_is_parent_roo Skip + ForkChoice - mainnet/altair/fork_choice/on_block/pyspec_tests/basic OK + ForkChoice - mainnet/altair/fork_choice/on_block/pyspec_tests/on_block_bad_parent_root OK ForkChoice - mainnet/altair/fork_choice/on_block/pyspec_tests/on_block_future_block Skip @@ -2842,6 +2848,8 @@ OK: 40/40 Fail: 0/40 Skip: 0/40 + ForkChoice - mainnet/bellatrix/fork_choice/get_head/pyspec_tests/proposer_boost_correct_he OK + ForkChoice - mainnet/bellatrix/fork_choice/get_head/pyspec_tests/shorter_chain_but_heavier OK + ForkChoice - mainnet/bellatrix/fork_choice/get_head/pyspec_tests/split_tie_breaker_no_atte OK + ForkChoice - mainnet/bellatrix/fork_choice/get_proposer_head/pyspec_tests/basic_is_head_ro Skip + ForkChoice - mainnet/bellatrix/fork_choice/get_proposer_head/pyspec_tests/basic_is_parent_ Skip + ForkChoice - mainnet/bellatrix/fork_choice/on_block/pyspec_tests/basic OK + ForkChoice - mainnet/bellatrix/fork_choice/on_block/pyspec_tests/on_block_bad_parent_root OK ForkChoice - mainnet/bellatrix/fork_choice/on_block/pyspec_tests/on_block_future_block Skip @@ -2852,6 +2860,7 @@ OK: 40/40 Fail: 0/40 Skip: 0/40 ForkChoice - mainnet/bellatrix/fork_choice/on_merge_block/pyspec_tests/block_lookup_failed Skip ForkChoice - mainnet/bellatrix/fork_choice/on_merge_block/pyspec_tests/too_early_for_merge Skip ForkChoice - mainnet/bellatrix/fork_choice/on_merge_block/pyspec_tests/too_late_for_merge Skip + ForkChoice - mainnet/bellatrix/fork_choice/should_override_forkchoice_update/pyspec_tests/ Skip + ForkChoice - mainnet/capella/fork_choice/ex_ante/pyspec_tests/ex_ante_attestations_is_grea OK + ForkChoice - mainnet/capella/fork_choice/ex_ante/pyspec_tests/ex_ante_sandwich_with_boost_ OK + ForkChoice - mainnet/capella/fork_choice/ex_ante/pyspec_tests/ex_ante_sandwich_with_honest OK @@ -2863,12 +2872,15 @@ OK: 40/40 Fail: 0/40 Skip: 0/40 + ForkChoice - mainnet/capella/fork_choice/get_head/pyspec_tests/proposer_boost_correct_head OK + ForkChoice - mainnet/capella/fork_choice/get_head/pyspec_tests/shorter_chain_but_heavier_w OK + ForkChoice - mainnet/capella/fork_choice/get_head/pyspec_tests/split_tie_breaker_no_attest OK + ForkChoice - mainnet/capella/fork_choice/get_proposer_head/pyspec_tests/basic_is_head_root Skip + ForkChoice - mainnet/capella/fork_choice/get_proposer_head/pyspec_tests/basic_is_parent_ro Skip + ForkChoice - mainnet/capella/fork_choice/on_block/pyspec_tests/basic OK + ForkChoice - mainnet/capella/fork_choice/on_block/pyspec_tests/on_block_bad_parent_root OK ForkChoice - mainnet/capella/fork_choice/on_block/pyspec_tests/on_block_future_block Skip + ForkChoice - mainnet/capella/fork_choice/on_block/pyspec_tests/proposer_boost OK + ForkChoice - mainnet/capella/fork_choice/on_block/pyspec_tests/proposer_boost_is_first_blo OK + ForkChoice - mainnet/capella/fork_choice/on_block/pyspec_tests/proposer_boost_root_same_sl OK + ForkChoice - mainnet/capella/fork_choice/should_override_forkchoice_update/pyspec_tests/sh Skip + ForkChoice - mainnet/deneb/fork_choice/ex_ante/pyspec_tests/ex_ante_attestations_is_greate OK + ForkChoice - mainnet/deneb/fork_choice/ex_ante/pyspec_tests/ex_ante_sandwich_with_boost_no OK + ForkChoice - mainnet/deneb/fork_choice/ex_ante/pyspec_tests/ex_ante_sandwich_with_honest_a OK @@ -2880,6 +2892,8 @@ OK: 40/40 Fail: 0/40 Skip: 0/40 + ForkChoice - mainnet/deneb/fork_choice/get_head/pyspec_tests/proposer_boost_correct_head OK + ForkChoice - mainnet/deneb/fork_choice/get_head/pyspec_tests/shorter_chain_but_heavier_wei OK + ForkChoice - mainnet/deneb/fork_choice/get_head/pyspec_tests/split_tie_breaker_no_attestat OK + ForkChoice - mainnet/deneb/fork_choice/get_proposer_head/pyspec_tests/basic_is_head_root Skip + ForkChoice - mainnet/deneb/fork_choice/get_proposer_head/pyspec_tests/basic_is_parent_root Skip + ForkChoice - mainnet/deneb/fork_choice/on_block/pyspec_tests/basic OK + ForkChoice - mainnet/deneb/fork_choice/on_block/pyspec_tests/invalid_data_unavailable OK + ForkChoice - mainnet/deneb/fork_choice/on_block/pyspec_tests/invalid_incorrect_proof OK @@ -2891,8 +2905,9 @@ OK: 40/40 Fail: 0/40 Skip: 0/40 + ForkChoice - mainnet/deneb/fork_choice/on_block/pyspec_tests/proposer_boost_is_first_block OK + ForkChoice - mainnet/deneb/fork_choice/on_block/pyspec_tests/proposer_boost_root_same_slot OK + ForkChoice - mainnet/deneb/fork_choice/on_block/pyspec_tests/simple_blob_data OK + ForkChoice - mainnet/deneb/fork_choice/should_override_forkchoice_update/pyspec_tests/shou Skip ``` -OK: 69/77 Fail: 0/77 Skip: 8/77 +OK: 69/88 Fail: 0/88 Skip: 19/88 ## Sync ```diff + Sync - mainnet/bellatrix/sync/optimistic/pyspec_tests/from_syncing_to_invalid OK @@ -2902,4 +2917,4 @@ OK: 69/77 Fail: 0/77 Skip: 8/77 OK: 3/3 Fail: 0/3 Skip: 0/3 ---TOTAL--- -OK: 2336/2344 Fail: 0/2344 Skip: 8/2344 +OK: 2336/2355 Fail: 0/2355 Skip: 19/2355 diff --git a/ConsensusSpecPreset-minimal.md b/ConsensusSpecPreset-minimal.md index decf1beca7..b60273090b 100644 --- a/ConsensusSpecPreset-minimal.md +++ b/ConsensusSpecPreset-minimal.md @@ -2355,7 +2355,6 @@ OK: 34/34 Fail: 0/34 Skip: 0/34 + Testing SignedBLSToExecutionChange OK + Testing SignedBeaconBlock OK + Testing SignedBeaconBlockHeader OK -+ Testing SignedBlobSidecar OK + Testing SignedContributionAndProof OK + Testing SignedVoluntaryExit OK + Testing SigningData OK @@ -2368,7 +2367,7 @@ OK: 34/34 Fail: 0/34 Skip: 0/34 + Testing VoluntaryExit OK + Testing Withdrawal OK ``` -OK: 49/49 Fail: 0/49 Skip: 0/49 +OK: 48/48 Fail: 0/48 Skip: 0/48 ## EF - Deneb - Sanity - Blocks [Preset: minimal] ```diff + [Invalid] EF - Deneb - Sanity - Blocks - invalid_all_zeroed_sig [Preset: minimal] OK @@ -2551,6 +2550,11 @@ OK: 20/20 Fail: 0/20 Skip: 0/20 + Light client - Update ranking - minimal/deneb/light_client/update_ranking/pyspec_tests/upd OK ``` OK: 4/4 Fail: 0/4 Skip: 0/4 +## EF - Merkle proof [Preset: minimal] +```diff ++ Merkle proof - Single merkle proof - minimal/deneb/merkle_proof/single_merkle_proof/Beacon OK +``` +OK: 1/1 Fail: 0/1 Skip: 0/1 ## EF - Phase 0 - Epoch Processing - Effective balance updates [Preset: minimal] ```diff + Effective balance updates - effective_balance_hysteresis [Preset: minimal] OK @@ -2977,6 +2981,8 @@ OK: 45/45 Fail: 0/45 Skip: 0/45 + ForkChoice - minimal/altair/fork_choice/get_head/pyspec_tests/split_tie_breaker_no_attesta OK + ForkChoice - minimal/altair/fork_choice/get_head/pyspec_tests/voting_source_beyond_two_epo OK + ForkChoice - minimal/altair/fork_choice/get_head/pyspec_tests/voting_source_within_two_epo OK + ForkChoice - minimal/altair/fork_choice/get_proposer_head/pyspec_tests/basic_is_head_root Skip + ForkChoice - minimal/altair/fork_choice/get_proposer_head/pyspec_tests/basic_is_parent_roo Skip + ForkChoice - minimal/altair/fork_choice/on_block/pyspec_tests/basic OK + ForkChoice - minimal/altair/fork_choice/on_block/pyspec_tests/incompatible_justification_u OK + ForkChoice - minimal/altair/fork_choice/on_block/pyspec_tests/incompatible_justification_u OK @@ -3023,6 +3029,8 @@ OK: 45/45 Fail: 0/45 Skip: 0/45 + ForkChoice - minimal/bellatrix/fork_choice/get_head/pyspec_tests/split_tie_breaker_no_atte OK + ForkChoice - minimal/bellatrix/fork_choice/get_head/pyspec_tests/voting_source_beyond_two_ OK + ForkChoice - minimal/bellatrix/fork_choice/get_head/pyspec_tests/voting_source_within_two_ OK + ForkChoice - minimal/bellatrix/fork_choice/get_proposer_head/pyspec_tests/basic_is_head_ro Skip + ForkChoice - minimal/bellatrix/fork_choice/get_proposer_head/pyspec_tests/basic_is_parent_ Skip + ForkChoice - minimal/bellatrix/fork_choice/on_block/pyspec_tests/basic OK + ForkChoice - minimal/bellatrix/fork_choice/on_block/pyspec_tests/incompatible_justificatio OK + ForkChoice - minimal/bellatrix/fork_choice/on_block/pyspec_tests/incompatible_justificatio OK @@ -3058,6 +3066,8 @@ OK: 45/45 Fail: 0/45 Skip: 0/45 + ForkChoice - minimal/bellatrix/fork_choice/reorg/pyspec_tests/simple_attempted_reorg_delay OK + ForkChoice - minimal/bellatrix/fork_choice/reorg/pyspec_tests/simple_attempted_reorg_delay OK + ForkChoice - minimal/bellatrix/fork_choice/reorg/pyspec_tests/simple_attempted_reorg_witho OK + ForkChoice - minimal/bellatrix/fork_choice/should_override_forkchoice_update/pyspec_tests/ Skip + ForkChoice - minimal/bellatrix/fork_choice/should_override_forkchoice_update/pyspec_tests/ Skip + ForkChoice - minimal/bellatrix/fork_choice/withholding/pyspec_tests/withholding_attack OK + ForkChoice - minimal/bellatrix/fork_choice/withholding/pyspec_tests/withholding_attack_unv OK + ForkChoice - minimal/capella/fork_choice/ex_ante/pyspec_tests/ex_ante_sandwich_with_honest OK @@ -3073,6 +3083,8 @@ OK: 45/45 Fail: 0/45 Skip: 0/45 + ForkChoice - minimal/capella/fork_choice/get_head/pyspec_tests/split_tie_breaker_no_attest OK + ForkChoice - minimal/capella/fork_choice/get_head/pyspec_tests/voting_source_beyond_two_ep OK + ForkChoice - minimal/capella/fork_choice/get_head/pyspec_tests/voting_source_within_two_ep OK + ForkChoice - minimal/capella/fork_choice/get_proposer_head/pyspec_tests/basic_is_head_root Skip + ForkChoice - minimal/capella/fork_choice/get_proposer_head/pyspec_tests/basic_is_parent_ro Skip + ForkChoice - minimal/capella/fork_choice/on_block/pyspec_tests/basic OK + ForkChoice - minimal/capella/fork_choice/on_block/pyspec_tests/incompatible_justification_ OK + ForkChoice - minimal/capella/fork_choice/on_block/pyspec_tests/incompatible_justification_ OK @@ -3104,6 +3116,8 @@ OK: 45/45 Fail: 0/45 Skip: 0/45 + ForkChoice - minimal/capella/fork_choice/reorg/pyspec_tests/simple_attempted_reorg_delayed OK + ForkChoice - minimal/capella/fork_choice/reorg/pyspec_tests/simple_attempted_reorg_delayed OK + ForkChoice - minimal/capella/fork_choice/reorg/pyspec_tests/simple_attempted_reorg_without OK + ForkChoice - minimal/capella/fork_choice/should_override_forkchoice_update/pyspec_tests/sh Skip + ForkChoice - minimal/capella/fork_choice/should_override_forkchoice_update/pyspec_tests/sh Skip + ForkChoice - minimal/capella/fork_choice/withholding/pyspec_tests/withholding_attack OK + ForkChoice - minimal/capella/fork_choice/withholding/pyspec_tests/withholding_attack_unvia OK + ForkChoice - minimal/deneb/fork_choice/ex_ante/pyspec_tests/ex_ante_sandwich_with_honest_a OK @@ -3119,6 +3133,8 @@ OK: 45/45 Fail: 0/45 Skip: 0/45 + ForkChoice - minimal/deneb/fork_choice/get_head/pyspec_tests/split_tie_breaker_no_attestat OK + ForkChoice - minimal/deneb/fork_choice/get_head/pyspec_tests/voting_source_beyond_two_epoc OK + ForkChoice - minimal/deneb/fork_choice/get_head/pyspec_tests/voting_source_within_two_epoc OK + ForkChoice - minimal/deneb/fork_choice/get_proposer_head/pyspec_tests/basic_is_head_root Skip + ForkChoice - minimal/deneb/fork_choice/get_proposer_head/pyspec_tests/basic_is_parent_root Skip + ForkChoice - minimal/deneb/fork_choice/on_block/pyspec_tests/basic OK + ForkChoice - minimal/deneb/fork_choice/on_block/pyspec_tests/incompatible_justification_up OK + ForkChoice - minimal/deneb/fork_choice/on_block/pyspec_tests/incompatible_justification_up OK @@ -3155,10 +3171,12 @@ OK: 45/45 Fail: 0/45 Skip: 0/45 + ForkChoice - minimal/deneb/fork_choice/reorg/pyspec_tests/simple_attempted_reorg_delayed_j OK + ForkChoice - minimal/deneb/fork_choice/reorg/pyspec_tests/simple_attempted_reorg_delayed_j OK + ForkChoice - minimal/deneb/fork_choice/reorg/pyspec_tests/simple_attempted_reorg_without_e OK + ForkChoice - minimal/deneb/fork_choice/should_override_forkchoice_update/pyspec_tests/shou Skip + ForkChoice - minimal/deneb/fork_choice/should_override_forkchoice_update/pyspec_tests/shou Skip + ForkChoice - minimal/deneb/fork_choice/withholding/pyspec_tests/withholding_attack OK + ForkChoice - minimal/deneb/fork_choice/withholding/pyspec_tests/withholding_attack_unviabl OK ``` -OK: 185/193 Fail: 0/193 Skip: 8/193 +OK: 185/207 Fail: 0/207 Skip: 22/207 ## Sync ```diff + Sync - minimal/bellatrix/sync/optimistic/pyspec_tests/from_syncing_to_invalid OK @@ -3168,4 +3186,4 @@ OK: 185/193 Fail: 0/193 Skip: 8/193 OK: 3/3 Fail: 0/3 Skip: 0/3 ---TOTAL--- -OK: 2578/2586 Fail: 0/2586 Skip: 8/2586 +OK: 2578/2600 Fail: 0/2600 Skip: 22/2600 diff --git a/beacon_chain/beacon_chain_db.nim b/beacon_chain/beacon_chain_db.nim index 2290505f51..88d96789d0 100644 --- a/beacon_chain/beacon_chain_db.nim +++ b/beacon_chain/beacon_chain_db.nim @@ -793,7 +793,8 @@ proc putBlock*( proc putBlobSidecar*( db: BeaconChainDB, value: BlobSidecar) = - db.blobs.putSZSSZ(blobkey(value.block_root, value.index), value) + let block_root = hash_tree_root(value.signed_block_header.message) + db.blobs.putSZSSZ(blobkey(block_root, value.index), value) proc delBlobSidecar*( db: BeaconChainDB, diff --git a/beacon_chain/consensus_object_pools/blob_quarantine.nim b/beacon_chain/consensus_object_pools/blob_quarantine.nim index cd62e1535e..0bf137129f 100644 --- a/beacon_chain/consensus_object_pools/blob_quarantine.nim +++ b/beacon_chain/consensus_object_pools/blob_quarantine.nim @@ -7,7 +7,7 @@ {.push raises: [].} -import ../spec/datatypes/deneb +import ../spec/helpers from std/sequtils import mapIt from std/strutils import join @@ -37,8 +37,9 @@ func put*(quarantine: var BlobQuarantine, blobSidecar: ref BlobSidecar) = oldest_blob_key = k break quarantine.blobs.del oldest_blob_key + let block_root = hash_tree_root(blobSidecar.signed_block_header.message) discard quarantine.blobs.hasKeyOrPut( - (blobSidecar.block_root, blobSidecar.index), blobSidecar) + (block_root, blobSidecar.index), blobSidecar) func blobIndices*(quarantine: BlobQuarantine, digest: Eth2Digest): seq[BlobIndex] = @@ -54,8 +55,9 @@ func hasBlob*( proposer_index: uint64, index: BlobIndex): bool = for blob_sidecar in quarantine.blobs.values: - if blob_sidecar.slot == slot and - blob_sidecar.proposer_index == proposer_index and + template block_header: untyped = blob_sidecar.signed_block_header.message + if block_header.slot == slot and + block_header.proposer_index == proposer_index and blob_sidecar.index == index: return true false diff --git a/beacon_chain/gossip_processing/eth2_processor.nim b/beacon_chain/gossip_processing/eth2_processor.nim index 87406275b5..f9a0acf404 100644 --- a/beacon_chain/gossip_processing/eth2_processor.nim +++ b/beacon_chain/gossip_processing/eth2_processor.nim @@ -245,7 +245,7 @@ proc processSignedBeaconBlock*( Opt.some(self.blobQuarantine[].popBlobs(signedBlock.root)) else: if not self.quarantine[].addBlobless(self.dag.finalizedHead.slot, - signedBlock): + signedBlock): notice "Block quarantine full (blobless)", blockRoot = shortLog(signedBlock.root), blck = shortLog(signedBlock.message), @@ -273,25 +273,26 @@ proc processSignedBeaconBlock*( v -proc processSignedBlobSidecar*( +proc processBlobSidecar*( self: var Eth2Processor, src: MsgSource, - signedBlobSidecar: deneb.SignedBlobSidecar, subnet_id: BlobId): ValidationRes = + blobSidecar: deneb.BlobSidecar, subnet_id: BlobId): ValidationRes = + template block_header: untyped = blobSidecar.signed_block_header.message + let wallTime = self.getCurrentBeaconTime() (afterGenesis, wallSlot) = wallTime.toSlot() logScope: - blob = shortLog(signedBlobSidecar.message) - signature = shortLog(signedBlobSidecar.signature) + blob = shortLog(blobSidecar) wallSlot # Potential under/overflows are fine; would just create odd metrics and logs - let delay = wallTime - signedBlobSidecar.message.slot.start_beacon_time + let delay = wallTime - block_header.slot.start_beacon_time debug "Blob received", delay let v = self.dag.validateBlobSidecar(self.quarantine, self.blobQuarantine, - signedBlobSidecar, wallTime, subnet_id) + blobSidecar, wallTime, subnet_id) if v.isErr(): debug "Dropping blob", error = v.error() @@ -299,21 +300,19 @@ proc processSignedBlobSidecar*( return v debug "Blob validated, putting in blob quarantine" - self.blobQuarantine[].put(newClone(signedBlobSidecar.message)) + self.blobQuarantine[].put(newClone(blobSidecar)) var skippedBlocks = false - if (let o = self.quarantine[].popBlobless( - signedBlobSidecar.message.block_root); o.isSome): + let block_root = hash_tree_root(block_header) + if (let o = self.quarantine[].popBlobless(block_root); o.isSome): let blobless = o.unsafeGet() if self.blobQuarantine[].hasBlobs(blobless): self.blockProcessor[].enqueueBlock( MsgSource.gossip, ForkedSignedBeaconBlock.init(blobless), - Opt.some(self.blobQuarantine[].popBlobs( - signedBlobSidecar.message.block_root)) - ) + Opt.some(self.blobQuarantine[].popBlobs(block_root))) else: discard self.quarantine[].addBlobless(self.dag.finalizedHead.slot, blobless) diff --git a/beacon_chain/gossip_processing/gossip_validation.nim b/beacon_chain/gossip_processing/gossip_validation.nim index 49248b832c..2f0770c6ca 100644 --- a/beacon_chain/gossip_processing/gossip_validation.nim +++ b/beacon_chain/gossip_processing/gossip_validation.nim @@ -182,6 +182,20 @@ func check_attestation_subnet( ok() +# https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.4/specs/deneb/p2p-interface.md#verify_blob_sidecar_inclusion_proof +func verify_blob_sidecar_inclusion_proof( + blob_sidecar: deneb.BlobSidecar): Result[void, ValidationError] = + let gindex = kzg_commitment_inclusion_proof_gindex(blob_sidecar.index) + if not is_valid_merkle_branch( + hash_tree_root(blob_sidecar.kzg_commitment), + blob_sidecar.kzg_commitment_inclusion_proof, + KZG_COMMITMENT_INCLUSION_PROOF_DEPTH, + get_subtree_index(gindex), + blob_sidecar.signed_block_header.message.body_root): + return errReject("BlobSidecar: inclusion proof not valid") + + ok() + # Gossip Validation # ---------------------------------------------------------------- @@ -302,93 +316,132 @@ template validateBeaconBlockBellatrix( # cannot occur here, because Nimbus's optimistic sync waits for either # `ACCEPTED` or `SYNCING` from the EL to get this far. -# https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.2/specs/deneb/p2p-interface.md#blob_sidecar_subnet_id +# https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.4/specs/deneb/p2p-interface.md#blob_sidecar_subnet_id proc validateBlobSidecar*( dag: ChainDAGRef, quarantine: ref Quarantine, - blobQuarantine: ref BlobQuarantine, sbs: SignedBlobSidecar, + blobQuarantine: ref BlobQuarantine, blob_sidecar: BlobSidecar, wallTime: BeaconTime, subnet_id: BlobId): Result[void, ValidationError] = # Some of the checks below have been reordered compared to the spec, to # perform the cheap checks first - in particular, we want to avoid loading # an `EpochRef` and checking signatures. This reordering might lead to # different IGNORE/REJECT results in turn affecting gossip scores. + template block_header: untyped = blob_sidecar.signed_block_header.message # [REJECT] The sidecar's index is consistent with `MAX_BLOBS_PER_BLOCK` # -- i.e. `blob_sidecar.index < MAX_BLOBS_PER_BLOCK` - if not (sbs.message.index < MAX_BLOBS_PER_BLOCK): - return dag.checkedReject("SignedBlobSidecar: index inconsistent") + if not (blob_sidecar.index < MAX_BLOBS_PER_BLOCK): + return dag.checkedReject("BlobSidecar: index inconsistent") # [REJECT] The sidecar is for the correct subnet -- i.e. # `compute_subnet_for_blob_sidecar(blob_sidecar.index) == subnet_id`. - if not (compute_subnet_for_blob_sidecar(sbs.message.index) == subnet_id): - return dag.checkedReject("SignedBlobSidecar: subnet incorrect") + if not (compute_subnet_for_blob_sidecar(blob_sidecar.index) == subnet_id): + return dag.checkedReject("BlobSidecar: subnet incorrect") # [IGNORE] The sidecar is not from a future slot (with a # `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. validate that # `block_header.slot <= current_slot` (a client MAY queue future sidecars - if not (sbs.message.slot <= + # for processing at the appropriate slot). + if not (block_header.slot <= (wallTime + MAXIMUM_GOSSIP_CLOCK_DISPARITY).slotOrZero): - return errIgnore("SignedBlobSidecar: slot too high") + return errIgnore("BlobSidecar: slot too high") - # [IGNORE] The block is from a slot greater than the latest - # finalized slot -- i.e. validate that - # signed_beacon_block.message.slot > - # compute_start_slot_at_epoch(state.finalized_checkpoint.epoch) - if not (sbs.message.slot > dag.finalizedHead.slot): - return errIgnore("SignedBlobSidecar: slot already finalized") + # [IGNORE] The sidecar is from a slot greater than the latest + # finalized slot -- i.e. validate that `block_header.slot > + # compute_start_slot_at_epoch(state.finalized_checkpoint.epoch)` + if not (block_header.slot > dag.finalizedHead.slot): + return errIgnore("BlobSidecar: slot already finalized") # [IGNORE] The sidecar is the first sidecar for the tuple # (block_header.slot, block_header.proposer_index, blob_sidecar.index) # with valid header signature, sidecar inclusion proof, and kzg proof. - if dag.getBlockRef(sbs.message.block_root).isSome(): - return errIgnore("SignedBlobSidecar: already have block") + let block_root = hash_tree_root(block_header) + if dag.getBlockRef(block_root).isSome(): + return errIgnore("BlobSidecar: already have block") if blobQuarantine[].hasBlob( - sbs.message.slot, sbs.message.proposer_index, sbs.message.index): - return errIgnore("SignedBlobSidecar: already have valid blob from same proposer") + block_header.slot, block_header.proposer_index, blob_sidecar.index): + return errIgnore("BlobSidecar: already have valid blob from same proposer") - # [IGNORE] The block's parent (defined by block.parent_root) has - # been seen (via both gossip and non-gossip sources) (a client MAY - # queue blocks for processing once the parent block is retrieved). - # [REJECT] The sidecar's block's parent (defined by sidecar.block_parent_root) - # passes validation. - let parentRes = dag.getBlockRef(sbs.message.block_parent_root) - if parentRes.isErr: - if sbs.message.block_parent_root in quarantine[].unviable: - return dag.checkedReject("SignedBlobSidecar: parent not validated") + # [REJECT] The sidecar's inclusion proof is valid as verified by + # `verify_blob_sidecar_inclusion_proof(blob_sidecar)`. + block: + let v = verify_blob_sidecar_inclusion_proof(blob_sidecar) + if v.isErr: + return dag.checkedReject(v.error) + + # [IGNORE] The sidecar's block's parent (defined by + # `block_header.parent_root`) has been seen (via both gossip and + # non-gossip sources) (a client MAY queue sidecars for processing + # once the parent block is retrieved). + # + # [REJECT] The sidecar's block's parent (defined by + # `block_header.parent_root`) passes validation. + let parent = dag.getBlockRef(block_header.parent_root).valueOr: + if block_header.parent_root in quarantine[].unviable: + quarantine[].addUnviable(block_root) + return dag.checkedReject("BlobSidecar: parent not validated") else: - return errIgnore("SignedBlobSidecar: parent not found") - template parent: untyped = parentRes.get + quarantine[].addMissing(block_header.parent_root) + return errIgnore("BlobSidecar: parent not found") # [REJECT] The sidecar is from a higher slot than the sidecar's - # block's parent (defined by sidecar.block_parent_root). - if sbs.message.slot <= parent.bid.slot: - return dag.checkedReject("SignedBlobSidecar: slot lower than parents'") + # block's parent (defined by `block_header.parent_root`). + if not (block_header.slot > parent.bid.slot): + return dag.checkedReject("BlobSidecar: slot lower than parents'") + + # [REJECT] The current finalized_checkpoint is an ancestor of the sidecar's + # block -- i.e. `get_checkpoint_block(store, block_header.parent_root, + # store.finalized_checkpoint.epoch) == store.finalized_checkpoint.root`. + let + finalized_checkpoint = getStateField(dag.headState, finalized_checkpoint) + ancestor = get_ancestor(parent, finalized_checkpoint.epoch.start_slot) + + if ancestor.isNil: + # This shouldn't happen: we should always be able to trace the parent back + # to the finalized checkpoint (else it wouldn't be in the DAG) + return errIgnore("BlobSidecar: Can't find ancestor") - # [REJECT] The sidecar is proposed by the expected proposer_index + if not ( + finalized_checkpoint.root == ancestor.root or + finalized_checkpoint.root.isZero): + quarantine[].addUnviable(block_root) + return dag.checkedReject( + "BlobSidecar: Finalized checkpoint not an ancestor") + + # [REJECT] The sidecar is proposed by the expected `proposer_index` # for the block's slot in the context of the current shuffling - # (defined by block_parent_root/slot). If the proposer_index - # cannot immediately be verified against the expected shuffling, - # the sidecar MAY be queued for later processing while proposers + # (defined by `block_header.parent_root`/`block_header.slot`). + # If the proposer_index cannot immediately be verified against the expected + # shuffling, the sidecar MAY be queued for later processing while proposers # for the block's branch are calculated -- in such a case do not # REJECT, instead IGNORE this message. - let - proposer = getProposer( - dag, parent, sbs.message.slot).valueOr: - warn "cannot compute proposer for blob" - return errIgnore("SignedBlobSidecar: Cannot compute proposer") - - if uint64(proposer) != sbs.message.proposer_index: - return dag.checkedReject("SignedBlobSidecar: Unexpected proposer") - - # [REJECT] The proposer signature, signed_blob_sidecar.signature, - # is valid as verified by verify_sidecar_signature. - if not verify_blob_signature( - dag.forkAtEpoch(sbs.message.slot.epoch), - getStateField(dag.headState, genesis_validators_root), - sbs.message.slot, - sbs.message, - dag.validatorKey(proposer).get(), - sbs.signature): - return dag.checkedReject("SignedBlobSidecar: invalid blob signature") + let proposer = getProposer(dag, parent, block_header.slot).valueOr: + warn "cannot compute proposer for blob" + return errIgnore("BlobSidecar: Cannot compute proposer") # internal issue + + if uint64(proposer) != block_header.proposer_index: + return dag.checkedReject("BlobSidecar: Unexpected proposer") + + # [REJECT] The proposer signature of `blob_sidecar.signed_block_header`, + # is valid with respect to the `block_header.proposer_index` pubkey. + if not verify_block_signature( + dag.forkAtEpoch(block_header.slot.epoch), + getStateField(dag.headState, genesis_validators_root), + block_header.slot, + block_root, + dag.validatorKey(proposer).get(), + blob_sidecar.signed_block_header.signature): + return dag.checkedReject("BlobSidecar: Invalid proposer signature") + + # [REJECT] The sidecar's blob is valid as verified by `verify_blob_kzg_proof( + # blob_sidecar.blob, blob_sidecar.kzg_commitment, blob_sidecar.kzg_proof)`. + block: + let ok = verifyProof( + blob_sidecar.blob, + blob_sidecar.kzg_commitment, + blob_sidecar.kzg_proof).valueOr: + return dag.checkedReject("BlobSidecar: blob verify failed") + if not ok: + return dag.checkedReject("BlobSidecar: blob invalid") ok() diff --git a/beacon_chain/networking/eth2_network.nim b/beacon_chain/networking/eth2_network.nim index 23f6c5ad82..25049c8a3d 100644 --- a/beacon_chain/networking/eth2_network.nim +++ b/beacon_chain/networking/eth2_network.nim @@ -2678,7 +2678,7 @@ proc broadcastBeaconBlock*( node.broadcast(topic, blck) proc broadcastBlobSidecar*( - node: Eth2Node, subnet_id: BlobId, blob: deneb.SignedBlobSidecar): + node: Eth2Node, subnet_id: BlobId, blob: deneb.BlobSidecar): Future[SendResult] = let forkPrefix = node.forkDigestAtEpoch(node.getWallEpoch) diff --git a/beacon_chain/nimbus_beacon_node.nim b/beacon_chain/nimbus_beacon_node.nim index 39c91cd110..ac90b41f69 100644 --- a/beacon_chain/nimbus_beacon_node.nim +++ b/beacon_chain/nimbus_beacon_node.nim @@ -1734,17 +1734,17 @@ proc installMessageValidators(node: BeaconNode) = when consensusFork >= ConsensusFork.Deneb: # blob_sidecar_{subnet_id} - # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.2/specs/deneb/p2p-interface.md#blob_sidecar_subnet_id + # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.4/specs/deneb/p2p-interface.md#blob_sidecar_subnet_id for it in BlobId: closureScope: # Needed for inner `proc`; don't lift it out of loop. let subnet_id = it node.network.addValidator( getBlobSidecarTopic(digest, subnet_id), proc ( - signedBlobSidecar: SignedBlobSidecar + blobSidecar: deneb.BlobSidecar ): ValidationResult = toValidationResult( - node.processor[].processSignedBlobSidecar( - MsgSource.gossip, signedBlobSidecar, subnet_id))) + node.processor[].processBlobSidecar( + MsgSource.gossip, blobSidecar, subnet_id))) node.installLightClientMessageValidators() diff --git a/beacon_chain/rpc/rest_beacon_api.nim b/beacon_chain/rpc/rest_beacon_api.nim index 56b741999b..944002f511 100644 --- a/beacon_chain/rpc/rest_beacon_api.nim +++ b/beacon_chain/rpc/rest_beacon_api.nim @@ -873,28 +873,29 @@ proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) = of ConsensusFork.Phase0: var blck = restBlock.phase0Data blck.root = hash_tree_root(blck.message) - await node.router.routeSignedBeaconBlock(blck, - Opt.none(SignedBlobSidecars)) + await node.router.routeSignedBeaconBlock( + blck, Opt.none(seq[BlobSidecar])) of ConsensusFork.Altair: var blck = restBlock.altairData blck.root = hash_tree_root(blck.message) - await node.router.routeSignedBeaconBlock(blck, - Opt.none(SignedBlobSidecars)) + await node.router.routeSignedBeaconBlock( + blck, Opt.none(seq[BlobSidecar])) of ConsensusFork.Bellatrix: var blck = restBlock.bellatrixData blck.root = hash_tree_root(blck.message) - await node.router.routeSignedBeaconBlock(blck, - Opt.none(SignedBlobSidecars)) + await node.router.routeSignedBeaconBlock( + blck, Opt.none(seq[BlobSidecar])) of ConsensusFork.Capella: var blck = restBlock.capellaData blck.root = hash_tree_root(blck.message) - await node.router.routeSignedBeaconBlock(blck, - Opt.none(SignedBlobSidecars)) + await node.router.routeSignedBeaconBlock( + blck, Opt.none(seq[BlobSidecar])) of ConsensusFork.Deneb: var blck = restBlock.denebData.signed_block blck.root = hash_tree_root(blck.message) await node.router.routeSignedBeaconBlock( - blck, Opt.some(asSeq restBlock.denebData.signed_blob_sidecars)) + blck, Opt.some(blck.create_blob_sidecars( + restBlock.denebData.kzg_proofs, restBlock.denebData.blobs))) if res.isErr(): return RestApiResponse.jsonError( @@ -948,28 +949,29 @@ proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) = of ConsensusFork.Phase0: var blck = restBlock.phase0Data blck.root = hash_tree_root(blck.message) - await node.router.routeSignedBeaconBlock(blck, - Opt.none(SignedBlobSidecars)) + await node.router.routeSignedBeaconBlock( + blck, Opt.none(seq[BlobSidecar])) of ConsensusFork.Altair: var blck = restBlock.altairData blck.root = hash_tree_root(blck.message) - await node.router.routeSignedBeaconBlock(blck, - Opt.none(SignedBlobSidecars)) + await node.router.routeSignedBeaconBlock( + blck, Opt.none(seq[BlobSidecar])) of ConsensusFork.Bellatrix: var blck = restBlock.bellatrixData blck.root = hash_tree_root(blck.message) - await node.router.routeSignedBeaconBlock(blck, - Opt.none(SignedBlobSidecars)) + await node.router.routeSignedBeaconBlock( + blck, Opt.none(seq[BlobSidecar])) of ConsensusFork.Capella: var blck = restBlock.capellaData blck.root = hash_tree_root(blck.message) - await node.router.routeSignedBeaconBlock(blck, - Opt.none(SignedBlobSidecars)) + await node.router.routeSignedBeaconBlock( + blck, Opt.none(seq[BlobSidecar])) of ConsensusFork.Deneb: var blck = restBlock.denebData.signed_block blck.root = hash_tree_root(blck.message) await node.router.routeSignedBeaconBlock( - blck, Opt.some(asSeq restBlock.denebData.signed_blob_sidecars)) + blck, Opt.some(blck.create_blob_sidecars( + restBlock.denebData.kzg_proofs, restBlock.denebData.blobs))) if res.isErr(): return RestApiResponse.jsonError( @@ -1066,8 +1068,8 @@ proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) = let res = withBlck(forked): forkyBlck.root = hash_tree_root(forkyBlck.message) - await node.router.routeSignedBeaconBlock(forkyBlck, - Opt.none(SignedBlobSidecars)) + await node.router.routeSignedBeaconBlock( + forkyBlck, Opt.none(seq[BlobSidecar])) if res.isErr(): return RestApiResponse.jsonError( diff --git a/beacon_chain/rpc/rest_validator_api.nim b/beacon_chain/rpc/rest_validator_api.nim index c016f899f7..734d974d51 100644 --- a/beacon_chain/rpc/rest_validator_api.nim +++ b/beacon_chain/rpc/rest_validator_api.nim @@ -407,26 +407,11 @@ proc installValidatorApiHandlers*(router: var RestRouter, node: BeaconNode) = withBlck(message.blck): let data = when consensusFork >= ConsensusFork.Deneb: - let bundle = message.blobsBundleOpt.get() - let blockRoot = hash_tree_root(forkyBlck) - var sidecars = newSeqOfCap[BlobSidecar](bundle.blobs.len) - for i in 0..= ConsensusFork.Deneb: + if kzg_proofs.isNone(): + reader.raiseUnexpectedValue("Field `kzg_proofs` is missing") + if blobs.isNone(): + reader.raiseUnexpectedValue("Field `blobs` is missing") + else: + if kzg_proofs.isSome(): + reader.raiseUnexpectedValue("Field `kzg_proofs` found but unsupported") + if blobs.isSome(): + reader.raiseUnexpectedValue("Field `blobs` found but unsupported") + case blck.kind of ConsensusFork.Phase0: value = RestPublishedSignedBlockContents( @@ -1865,7 +1885,8 @@ proc readValue*(reader: var JsonReader[RestJson], denebData: DenebSignedBlockContents( # Constructed to be internally consistent signed_block: signed_message.get().distinctBase.denebData, - signed_blob_sidecars: signed_blob_sidecars.get() + kzg_proofs: kzg_proofs.get(), + blobs: blobs.get() ) ) diff --git a/beacon_chain/spec/eth2_apis/rest_types.nim b/beacon_chain/spec/eth2_apis/rest_types.nim index c36d25a610..9280cb6668 100644 --- a/beacon_chain/spec/eth2_apis/rest_types.nim +++ b/beacon_chain/spec/eth2_apis/rest_types.nim @@ -317,7 +317,8 @@ type DenebSignedBlockContents* = object signed_block*: deneb.SignedBeaconBlock - signed_blob_sidecars*: List[SignedBlobSidecar, Limit MAX_BLOBS_PER_BLOCK] + kzg_proofs*: deneb.KzgProofs + blobs*: deneb.Blobs RestPublishedSignedBlockContents* = object case kind*: ConsensusFork @@ -339,7 +340,8 @@ type DenebBlockContents* = object `block`*: deneb.BeaconBlock - blob_sidecars*: List[BlobSidecar, Limit MAX_BLOBS_PER_BLOCK] + kzg_proofs*: deneb.KzgProofs + blobs*: deneb.Blobs ProduceBlockResponseV2* = object case kind*: ConsensusFork diff --git a/beacon_chain/spec/forks.nim b/beacon_chain/spec/forks.nim index 1360cd2f0e..8dc3d39ad8 100644 --- a/beacon_chain/spec/forks.nim +++ b/beacon_chain/spec/forks.nim @@ -905,8 +905,9 @@ template withStateAndBlck*( func toBeaconBlockHeader*( blck: SomeForkyBeaconBlock | - capella_mev.BlindedBeaconBlock | deneb_mev.BlindedBeaconBlock): - BeaconBlockHeader = + capella_mev.BlindedBeaconBlock | + deneb_mev.BlindedBeaconBlock +): BeaconBlockHeader = ## Reduce a given `BeaconBlock` to its `BeaconBlockHeader`. BeaconBlockHeader( slot: blck.slot, @@ -918,7 +919,7 @@ func toBeaconBlockHeader*( template toBeaconBlockHeader*( blck: SomeForkySignedBeaconBlock): BeaconBlockHeader = ## Reduce a given `SignedBeaconBlock` to its `BeaconBlockHeader`. - blck.message.toBeaconBlockHeader + blck.message.toBeaconBlockHeader() template toBeaconBlockHeader*( blckParam: ForkedMsgTrustedSignedBeaconBlock | @@ -926,6 +927,16 @@ template toBeaconBlockHeader*( ## Reduce a given signed beacon block to its `BeaconBlockHeader`. withBlck(blckParam): forkyBlck.toBeaconBlockHeader() +func toSignedBeaconBlockHeader*( + signedBlock: SomeForkySignedBeaconBlock | + capella_mev.SignedBlindedBeaconBlock | + deneb_mev.SignedBlindedBeaconBlock +): SignedBeaconBlockHeader = + ## Reduce a given `SignedBeaconBlock` to its `SignedBeaconBlockHeader`. + SignedBeaconBlockHeader( + message: signedBlock.message.toBeaconBlockHeader(), + signature: signedBlock.signature) + func genesisFork*(cfg: RuntimeConfig): Fork = Fork( previous_version: cfg.GENESIS_FORK_VERSION, diff --git a/beacon_chain/spec/helpers.nim b/beacon_chain/spec/helpers.nim index b392d612ea..dc6c448466 100644 --- a/beacon_chain/spec/helpers.nim +++ b/beacon_chain/spec/helpers.nim @@ -211,6 +211,54 @@ func has_flag*(flags: ParticipationFlags, flag_index: TimelyFlag): bool = let flag = ParticipationFlags(1'u8 shl ord(flag_index)) (flags and flag) == flag +func create_blob_sidecars*( + forkyBlck: deneb.SignedBeaconBlock, + kzg_proofs: KzgProofs, + blobs: Blobs): seq[BlobSidecar] = + template kzg_commitments: untyped = + forkyBlck.message.body.blob_kzg_commitments + doAssert kzg_proofs.len == blobs.len + doAssert kzg_proofs.len == kzg_commitments.len + + var res = newSeqOfCap[BlobSidecar](blobs.len) + let signedBlockHeader = forkyBlck.toSignedBeaconBlockHeader() + for i in 0 ..< blobs.lenu64: + var sidecar = BlobSidecar( + index: i, + blob: blobs[i], + kzg_commitment: kzg_commitments[i], + kzg_proof: kzg_proofs[i], + signed_block_header: signedBlockHeader) + forkyBlck.message.body.build_proof( + kzg_commitment_inclusion_proof_gindex(i), + sidecar.kzg_commitment_inclusion_proof).expect("Valid gindex") + res.add(sidecar) + res + +func create_blob_sidecars*( + forkyBlck: deneb_mev.SignedBlindedBeaconBlock, + kzg_proofs: KzgProofs, + blob_roots: BlobRoots): seq[BlindedBlobSidecar] = + template kzg_commitments: untyped = + forkyBlck.message.body.blob_kzg_commitments + doAssert kzg_proofs.len == blob_roots.len + doAssert kzg_proofs.len == kzg_commitments.len + + var res = newSeqOfCap[BlindedBlobSidecar](blob_roots.len) + let signedBlockHeader = forkyBlck.toSignedBeaconBlockHeader() + for i in 0 ..< blob_roots.lenu64: + var sidecar = BlindedBlobSidecar( + index: i, + blob_root: blob_roots[i], + kzg_commitment: kzg_commitments[i], + kzg_proof: kzg_proofs[i], + signed_block_header: signedBlockHeader) + forkyBlck.message.body.build_proof( + kzg_commitment_inclusion_proof_gindex(i), + sidecar.kzg_commitment_inclusion_proof).expect("Valid gindex") + res.add(sidecar) + res + # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/altair/light-client/sync-protocol.md#is_sync_committee_update template is_sync_committee_update*(update: SomeForkyLightClientUpdate): bool = when update is SomeForkyLightClientUpdateWithSyncCommittee: diff --git a/beacon_chain/spec/mev/deneb_mev.nim b/beacon_chain/spec/mev/deneb_mev.nim index 736ed0bf73..f64f87860c 100644 --- a/beacon_chain/spec/mev/deneb_mev.nim +++ b/beacon_chain/spec/mev/deneb_mev.nim @@ -69,25 +69,18 @@ type # https://github.com/ethereum/builder-specs/blob/534e4f81276b8346d785ed9aba12c4c74b927ec6/specs/deneb/builder.md#blindedblobsidecar BlindedBlobSidecar* = object - block_root*: Eth2Digest index*: uint64 - slot*: uint64 - block_parent_root*: Eth2Digest - proposer_index*: uint64 blob_root*: Eth2Digest kzg_commitment*: KZGCommitment kzg_proof*: KZGProof - - # https://github.com/ethereum/builder-specs/blob/534e4f81276b8346d785ed9aba12c4c74b927ec6/specs/deneb/builder.md#signedblindedblobsidecar - SignedBlindedBlobSidecar* = object - message*: BlindedBlobSidecar - signature*: ValidatorSig + signed_block_header*: SignedBeaconBlockHeader + kzg_commitment_inclusion_proof*: + array[KZG_COMMITMENT_INCLUSION_PROOF_DEPTH, Eth2Digest] # https://github.com/ethereum/builder-specs/blob/534e4f81276b8346d785ed9aba12c4c74b927ec6/specs/deneb/builder.md#signedblindedblockcontents SignedBlindedBeaconBlockContents* = object signed_blinded_block*: deneb_mev.SignedBlindedBeaconBlock - signed_blinded_blob_sidecars*: - List[SignedBlindedBlobSidecar, Limit MAX_BLOBS_PER_BLOCK] + blinded_blob_sidecars*: List[BlindedBlobSidecar, Limit MAX_BLOBS_PER_BLOCK] # https://github.com/ethereum/builder-specs/blob/534e4f81276b8346d785ed9aba12c4c74b927ec6/specs/deneb/builder.md#executionpayloadandblobsbundle ExecutionPayloadAndBlobsBundle* = object diff --git a/beacon_chain/spec/signatures.nim b/beacon_chain/spec/signatures.nim index 1aa318537e..417c751223 100644 --- a/beacon_chain/spec/signatures.nim +++ b/beacon_chain/spec/signatures.nim @@ -88,15 +88,6 @@ func compute_block_signing_root*( fork, DOMAIN_BEACON_PROPOSER, epoch, genesis_validators_root) compute_signing_root(blck, domain) -func compute_blob_signing_root( - fork: Fork, genesis_validators_root: Eth2Digest, slot: Slot, - blob: BlindedBlobSidecar | BlobSidecar): Eth2Digest = - let - epoch = epoch(slot) - domain = get_domain(fork, DOMAIN_BLOB_SIDECAR, epoch, - genesis_validators_root) - compute_signing_root(blob, domain) - # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/phase0/validator.md#signature func get_block_signature*( fork: Fork, genesis_validators_root: Eth2Digest, slot: Slot, @@ -106,16 +97,6 @@ func get_block_signature*( blsSign(privkey, signing_root.data) -# https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/deneb/validator.md#constructing-the-signedblobsidecars -proc get_blob_sidecar_signature*( - fork: Fork, genesis_validators_root: Eth2Digest, slot: Slot, - blob: BlindedBlobSidecar | BlobSidecar, privkey: ValidatorPrivKey): - CookedSig = - let signing_root = compute_blob_signing_root( - fork, genesis_validators_root, slot, blob) - - blsSign(privkey, signing_root.data) - proc verify_block_signature*( fork: Fork, genesis_validators_root: Eth2Digest, slot: Slot, blck: Eth2Digest | SomeForkyBeaconBlock | BeaconBlockHeader, @@ -127,17 +108,6 @@ proc verify_block_signature*( blsVerify(pubkey, signing_root.data, signature) -proc verify_blob_signature*( - fork: Fork, genesis_validators_root: Eth2Digest, slot: Slot, - blobSidecar: BlobSidecar, - pubkey: ValidatorPubKey | CookedPubKey, signature: SomeSig): bool = - withTrust(signature): - let - signing_root = compute_blob_signing_root( - fork, genesis_validators_root, slot, blobSidecar) - - blsVerify(pubkey, signing_root.data, signature) - func compute_aggregate_and_proof_signing_root*( fork: Fork, genesis_validators_root: Eth2Digest, aggregate_and_proof: AggregateAndProof): Eth2Digest = diff --git a/beacon_chain/sync/request_manager.nim b/beacon_chain/sync/request_manager.nim index d37a13d504..045943e44d 100644 --- a/beacon_chain/sync/request_manager.nim +++ b/beacon_chain/sync/request_manager.nim @@ -93,9 +93,10 @@ proc checkResponse(idList: seq[BlobIdentifier], if len(blobs) > len(idList): return false for blob in blobs: + let block_root = hash_tree_root(blob.signed_block_header.message) var found = false for id in idList: - if id.block_root == blob.block_root and + if id.block_root == block_root and id.index == blob.index: found = true break @@ -204,8 +205,9 @@ proc fetchBlobsFromNetwork(self: RequestManager, self.blobQuarantine[].put(b) var curRoot: Eth2Digest for b in ublobs: - if b.block_root != curRoot: - curRoot = b.block_root + let block_root = hash_tree_root(b.signed_block_header.message) + if block_root != curRoot: + curRoot = block_root if (let o = self.quarantine[].popBlobless(curRoot); o.isSome): let b = o.unsafeGet() discard await self.blockVerifier(ForkedSignedBeaconBlock.init(b), false) diff --git a/beacon_chain/sync/sync_manager.nim b/beacon_chain/sync/sync_manager.nim index 2bd07a20ae..c0c35567ee 100644 --- a/beacon_chain/sync/sync_manager.nim +++ b/beacon_chain/sync/sync_manager.nim @@ -247,9 +247,9 @@ func groupBlobs*[T](req: SyncRequest[T], # reached end of blobs, have more blobless blocks break for blob in blobs[blobCursor..len(blobs)-1]: - if blob.slot < slot: + if blob.signed_block_header.message.slot < slot: return Result[seq[BlobSidecars], string].err "invalid blob sequence" - if blob.slot==slot: + if blob.signed_block_header.message.slot == slot: grouped[i].add(blob) blobCursor = blobCursor + 1 i = i + 1 @@ -439,7 +439,7 @@ proc syncStep[A, B](man: SyncManager[A, B], index: int, peer: A) {.async.} = blobs_map = blobSmap, request = req if len(blobData) > 0: - let slots = mapIt(blobData, it[].slot) + let slots = mapIt(blobData, it[].signed_block_header.message.slot) let uniqueSlots = foldl(slots, combine(a, b), @[slots[0]]) if not(checkResponse(req, uniqueSlots)): peer.updateScore(PeerScoreBadResponse) @@ -464,7 +464,8 @@ proc syncStep[A, B](man: SyncManager[A, B], index: int, peer: A) {.async.} = man.queue.push(req) return for i, blk in blockData: - if len(blobs[i]) > 0 and blk[].slot != blobs[i][0].slot: + if len(blobs[i]) > 0 and blk[].slot != + blobs[i][0].signed_block_header.message.slot: peer.updateScore(PeerScoreNoValues) man.queue.push(req) debug "block and blobs data have inconsistent slots" diff --git a/beacon_chain/sync/sync_queue.nim b/beacon_chain/sync/sync_queue.nim index 9e1f306cbe..fb2fbb7628 100644 --- a/beacon_chain/sync/sync_queue.nim +++ b/beacon_chain/sync/sync_queue.nim @@ -119,9 +119,9 @@ proc getShortMap*[T](req: SyncRequest[T], if cur >= lenu64(data): res.add('|') continue - if slot == data[cur].slot: + if slot == data[cur].signed_block_header.message.slot: for k in cur..= lenu64(data) or slot != data[k].slot: + if k >= lenu64(data) or slot != data[k].signed_block_header.message.slot: res.add('|') break else: diff --git a/beacon_chain/validator_client/block_service.nim b/beacon_chain/validator_client/block_service.nim index 995d3c377e..a054647775 100644 --- a/beacon_chain/validator_client/block_service.nim +++ b/beacon_chain/validator_client/block_service.nim @@ -21,12 +21,11 @@ const logScope: service = ServiceName type - BlobList = List[BlobSidecar, Limit MAX_BLOBS_PER_BLOCK] - PreparedBeaconBlock = object blockRoot*: Eth2Digest data*: ForkedBeaconBlock - blobsOpt*: Opt[BlobList] + kzgProofsOpt*: Opt[deneb.KzgProofs] + blobsOpt*: Opt[deneb.Blobs] PreparedBlindedBeaconBlock = object blockRoot*: Eth2Digest @@ -66,27 +65,34 @@ proc produceBlock( let blck = produceBlockResponse.phase0Data return Opt.some(PreparedBeaconBlock(blockRoot: hash_tree_root(blck), data: ForkedBeaconBlock.init(blck), - blobsOpt: Opt.none(BlobList))) + kzgProofsOpt: Opt.none(deneb.KzgProofs), + blobsOpt: Opt.none(deneb.Blobs))) of ConsensusFork.Altair: let blck = produceBlockResponse.altairData return Opt.some(PreparedBeaconBlock(blockRoot: hash_tree_root(blck), data: ForkedBeaconBlock.init(blck), - blobsOpt: Opt.none(BlobList))) + kzgProofsOpt: Opt.none(deneb.KzgProofs), + blobsOpt: Opt.none(deneb.Blobs))) of ConsensusFork.Bellatrix: let blck = produceBlockResponse.bellatrixData return Opt.some(PreparedBeaconBlock(blockRoot: hash_tree_root(blck), data: ForkedBeaconBlock.init(blck), - blobsOpt: Opt.none(BlobList))) + kzgProofsOpt: Opt.none(deneb.KzgProofs), + blobsOpt: Opt.none(deneb.Blobs))) of ConsensusFork.Capella: let blck = produceBlockResponse.capellaData return Opt.some(PreparedBeaconBlock(blockRoot: hash_tree_root(blck), data: ForkedBeaconBlock.init(blck), - blobsOpt: Opt.none(BlobList))) + kzgProofsOpt: Opt.none(deneb.KzgProofs), + blobsOpt: Opt.none(deneb.Blobs))) of ConsensusFork.Deneb: - let blck = produceBlockResponse.denebData.`block` - let blobs = produceBlockResponse.denebData.blob_sidecars + let + blck = produceBlockResponse.denebData.`block` + kzgProofs = produceBlockResponse.denebData.kzg_proofs + blobs = produceBlockResponse.denebData.blobs return Opt.some(PreparedBeaconBlock(blockRoot: hash_tree_root(blck), data: ForkedBeaconBlock.init(blck), + kzgProofsOpt: Opt.some(kzgProofs), blobsOpt: Opt.some(blobs))) proc produceBlindedBlock( @@ -388,30 +394,14 @@ proc publishBlock(vc: ValidatorClientRef, currentSlot, slot: Slot, root: preparedBlock.blockRoot, signature: signature)) of ConsensusFork.Deneb: - let blobs = preparedBlock.blobsOpt.get() - var signed: seq[SignedBlobSidecar] = @[] - for i in 0..= ConsensusFork.Deneb: - var sidecars: seq[BlobSidecar] - let bundle = collectedBids.engineBlockFut.read.get().blobsBundleOpt.get - let (blobs, commitments, proofs) = ( - bundle.blobs, bundle.commitments, bundle.proofs) - for i in 0..= ConsensusFork.Deneb: + template blobsBundle: untyped = + collectedBids.engineBlockFut.read.get.blobsBundleOpt.get + Opt.some(signedBlock.create_blob_sidecars( + blobsBundle.proofs, blobsBundle.blobs)) else: - static: doAssert "Unknown SignedBeaconBlock type" - - newBlockRef = - (await node.router.routeSignedBeaconBlock(signedBlock, signedBlobs)).valueOr: - return head # Errors logged in router + Opt.none(seq[BlobSidecar]) + newBlockRef = ( + await node.router.routeSignedBeaconBlock(signedBlock, blobsOpt) + ).valueOr: + return head # Errors logged in router if newBlockRef.isNone(): return head # Validation errors logged in router diff --git a/beacon_chain/validators/message_router.nim b/beacon_chain/validators/message_router.nim index 431099ff3b..86dc5614df 100644 --- a/beacon_chain/validators/message_router.nim +++ b/beacon_chain/validators/message_router.nim @@ -82,14 +82,10 @@ template blockProcessor(router: MessageRouter): ref BlockProcessor = template getCurrentBeaconTime(router: MessageRouter): BeaconTime = router.processor[].getCurrentBeaconTime() -type SignedBlobSidecars* = seq[SignedBlobSidecar] -func shortLog*(v: SignedBlobSidecars): auto = - "[" & v.mapIt(shortLog(it)).join(", ") & "]" - type RouteBlockResult = Result[Opt[BlockRef], cstring] proc routeSignedBeaconBlock*( router: ref MessageRouter, blck: ForkySignedBeaconBlock, - blobsOpt: Opt[SignedBlobSidecars]): Future[RouteBlockResult] {.async.} = + blobsOpt: Opt[seq[BlobSidecar]]): Future[RouteBlockResult] {.async.} = ## Validate and broadcast beacon block, then add it to the block database ## Returns the new Head when block is added successfully to dag, none when ## block passes validation but is not added, and error otherwise @@ -112,8 +108,8 @@ proc routeSignedBeaconBlock*( let blobs = blobsOpt.get() let kzgCommits = blck.message.body.blob_kzg_commitments.asSeq if blobs.len > 0 or kzgCommits.len > 0: - let res = validate_blobs(kzgCommits, blobs.mapIt(it.message.blob), - blobs.mapIt(it.message.kzg_proof)) + let res = validate_blobs(kzgCommits, blobs.mapIt(it.blob), + blobs.mapIt(it.kzg_proof)) if res.isErr(): warn "blobs failed validation", blockRoot = shortLog(blck.root), @@ -145,26 +141,26 @@ proc routeSignedBeaconBlock*( blockRoot = shortLog(blck.root), blck = shortLog(blck.message), signature = shortLog(blck.signature), error = res.error() - var blobs = Opt.none(seq[ref BlobSidecar]) + var blobRefs = Opt.none(BlobSidecars) if blobsOpt.isSome(): - let signedBlobs = blobsOpt.get() - var workers = newSeq[Future[SendResult]](signedBlobs.len) - for i in 0.. Date: Mon, 6 Nov 2023 12:10:36 +0100 Subject: [PATCH 33/35] do not download Holesky genesis on `git clone` (#5573) Holesky genesis.ssz file may be unavailable due to quota limits on `eth-clients/holesky`; do not download it by default during checkout. Nimbus will download it on first startup from a mirror instead. --- Makefile | 4 ++-- ci/Jenkinsfile.benchmarks | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index a11259c6b5..26c29d38a2 100644 --- a/Makefile +++ b/Makefile @@ -118,9 +118,9 @@ ifneq ($(OS), Windows_NT) PLATFORM_SPECIFIC_TARGETS += gnosis-build endif -# We don't need the `vendor/holesky/public-keys/all.txt` file but fetching it +# We don't need these `vendor/holesky` files but fetching them # may trigger 'This repository is over its data quota' from GitHub -GIT_SUBMODULE_CONFIG := -c lfs.fetchexclude=/public-keys/all.txt +GIT_SUBMODULE_CONFIG := -c lfs.fetchexclude=/public-keys/all.txt,/custom_config_data/genesis.ssz,/custom_config_data/parsedBeaconState.json ifeq ($(NIM_PARAMS),) # "variables.mk" was not included, so we update the submodules. diff --git a/ci/Jenkinsfile.benchmarks b/ci/Jenkinsfile.benchmarks index e584925903..481b84b421 100644 --- a/ci/Jenkinsfile.benchmarks +++ b/ci/Jenkinsfile.benchmarks @@ -23,7 +23,7 @@ node("metal") { /* source code checkout */ checkout scm /* we need to update the submodules before caching kicks in */ - sh "git -c lfs.fetchexclude=/public-keys/all.txt submodule update --init --recursive" + sh "git -c lfs.fetchexclude=/public-keys/all.txt,/custom_config_data/genesis.ssz,/custom_config_data/parsedBeaconState.json submodule update --init --recursive" } stage("Build") { From 49c851109ea41e9fcb59d4adeb7187ee4eaf6cb5 Mon Sep 17 00:00:00 2001 From: Eugene Kabanov Date: Mon, 6 Nov 2023 16:40:44 +0200 Subject: [PATCH 34/35] VC: Add pruning slashing database. (#5551) * Add slashing database pruning to VC. Fix GetBlockHeaderResponse object declaration (spec has been changed). * Switch to getFinalizedBlockHeader instead. * Fix proper sign. Add statements. Show pruning log statement only when pruning happens. * Optimize and remove debugging helpers. --- .../eth2_apis/eth2_rest_serialization.nim | 1 + beacon_chain/spec/eth2_apis/rest_types.nim | 7 +- beacon_chain/validator_client/api.nim | 89 +++++++++++++++++++ beacon_chain/validator_client/common.nim | 3 + .../validator_client/duties_service.nim | 75 +++++++++++++++- 5 files changed, 173 insertions(+), 2 deletions(-) diff --git a/beacon_chain/spec/eth2_apis/eth2_rest_serialization.nim b/beacon_chain/spec/eth2_apis/eth2_rest_serialization.nim index 1fd672f1a2..546bb2d382 100644 --- a/beacon_chain/spec/eth2_apis/eth2_rest_serialization.nim +++ b/beacon_chain/spec/eth2_apis/eth2_rest_serialization.nim @@ -130,6 +130,7 @@ type DataRootEnclosedObject | DataOptimisticObject | DataVersionEnclosedObject | + DataOptimisticAndFinalizedObject | GetBlockV2Response | GetDistributedKeystoresResponse | GetKeystoresResponse | diff --git a/beacon_chain/spec/eth2_apis/rest_types.nim b/beacon_chain/spec/eth2_apis/rest_types.nim index 9280cb6668..b442a495c2 100644 --- a/beacon_chain/spec/eth2_apis/rest_types.nim +++ b/beacon_chain/spec/eth2_apis/rest_types.nim @@ -395,6 +395,11 @@ type data*: T execution_optimistic*: Option[bool] + DataOptimisticAndFinalizedObject*[T] = object + data*: T + execution_optimistic*: Option[bool] + finalized*: Option[bool] + ForkedSignedBlockHeader* = object message*: uint32 # message offset signature*: ValidatorSig @@ -515,7 +520,7 @@ type GetAggregatedAttestationResponse* = DataEnclosedObject[Attestation] GetAttesterDutiesResponse* = DataRootEnclosedObject[seq[RestAttesterDuty]] GetBlockAttestationsResponse* = DataEnclosedObject[seq[Attestation]] - GetBlockHeaderResponse* = DataOptimisticObject[RestBlockHeaderInfo] + GetBlockHeaderResponse* = DataOptimisticAndFinalizedObject[RestBlockHeaderInfo] GetBlockHeadersResponse* = DataEnclosedObject[seq[RestBlockHeaderInfo]] GetBlockRootResponse* = DataOptimisticObject[RestRoot] GetDebugChainHeadsResponse* = DataEnclosedObject[seq[RestChainHead]] diff --git a/beacon_chain/validator_client/api.nim b/beacon_chain/validator_client/api.nim index 89855588be..9304a73aa5 100644 --- a/beacon_chain/validator_client/api.nim +++ b/beacon_chain/validator_client/api.nim @@ -2570,3 +2570,92 @@ proc getValidatorsLiveness*( res return GetValidatorsLivenessResponse(data: response) + +proc getFinalizedBlockHeader*( + vc: ValidatorClientRef, + ): Future[Opt[GetBlockHeaderResponse]] {.async.} = + const RequestName = "getFinalizedBlockHeader" + + let + blockIdent = BlockIdent.init(BlockIdentType.Finalized) + resp = vc.onceToAll(RestPlainResponse, + SlotDuration, + ViableNodeStatus, + {BeaconNodeRole.Duties}, + getBlockHeaderPlain(it, blockIdent)) + case resp.status + of ApiOperation.Timeout: + debug "Unable to obtain finalized block header in time", + timeout = SlotDuration + return Opt.none(GetBlockHeaderResponse) + of ApiOperation.Interrupt: + debug "Finalized block header request was interrupted" + return Opt.none(GetBlockHeaderResponse) + of ApiOperation.Failure: + debug "Unexpected error happened while trying to get finalized block header" + return Opt.none(GetBlockHeaderResponse) + of ApiOperation.Success: + var oldestBlockHeader: GetBlockHeaderResponse + var oldestEpoch: Opt[Epoch] + for apiResponse in resp.data: + if apiResponse.data.isErr(): + debug "Unable to get finalized block header", + endpoint = apiResponse.node, error = apiResponse.data.error + else: + let response = apiResponse.data.get() + case response.status + of 200: + let res = decodeBytes(GetBlockHeaderResponse, + response.data, response.contentType) + if res.isOk(): + let + rdata = res.get() + epoch = rdata.data.header.message.slot.epoch() + if oldestEpoch.get(FAR_FUTURE_EPOCH) > epoch: + oldestEpoch = Opt.some(epoch) + oldestBlockHeader = rdata + else: + let failure = ApiNodeFailure.init( + ApiFailure.UnexpectedResponse, RequestName, + apiResponse.node, response.status, $res.error) + # We do not update beacon node's status anymore because of + # issue #5377. + debug ResponseDecodeError, reason = getFailureReason(failure) + continue + of 400: + let failure = ApiNodeFailure.init( + ApiFailure.Invalid, RequestName, + apiResponse.node, response.status, response.getErrorMessage()) + # We do not update beacon node's status anymore because of + # issue #5377. + debug ResponseInvalidError, reason = getFailureReason(failure) + continue + of 404: + let failure = ApiNodeFailure.init( + ApiFailure.NotFound, RequestName, + apiResponse.node, response.status, response.getErrorMessage()) + # We do not update beacon node's status anymore because of + # issue #5377. + debug ResponseNotFoundError, reason = getFailureReason(failure) + continue + of 500: + let failure = ApiNodeFailure.init( + ApiFailure.Internal, RequestName, + apiResponse.node, response.status, response.getErrorMessage()) + # We do not update beacon node's status anymore because of + # issue #5377. + debug ResponseInternalError, reason = getFailureReason(failure) + continue + else: + let failure = ApiNodeFailure.init( + ApiFailure.UnexpectedCode, RequestName, + apiResponse.node, response.status, response.getErrorMessage()) + # We do not update beacon node's status anymore because of + # issue #5377. + debug ResponseUnexpectedError, reason = getFailureReason(failure) + continue + + if oldestEpoch.isSome(): + return Opt.some(oldestBlockHeader) + else: + return Opt.none(GetBlockHeaderResponse) diff --git a/beacon_chain/validator_client/common.nim b/beacon_chain/validator_client/common.nim index eb4a505571..2d6dd07670 100644 --- a/beacon_chain/validator_client/common.nim +++ b/beacon_chain/validator_client/common.nim @@ -65,7 +65,9 @@ type DutiesServiceRef* = ref object of ClientServiceRef pollingAttesterDutiesTask*: Future[void] pollingSyncDutiesTask*: Future[void] + pruneSlashingDatabaseTask*: Future[void] syncSubscriptionEpoch*: Opt[Epoch] + lastSlashingEpoch*: Opt[Epoch] FallbackServiceRef* = ref object of ClientServiceRef changesEvent*: AsyncEvent @@ -229,6 +231,7 @@ type blocksSeen*: Table[Slot, BlockDataItem] rootsSeen*: Table[Eth2Digest, Slot] processingDelay*: Opt[Duration] + finalizedEpoch*: Opt[Epoch] rng*: ref HmacDrbgContext ApiStrategyKind* {.pure.} = enum diff --git a/beacon_chain/validator_client/duties_service.nim b/beacon_chain/validator_client/duties_service.nim index fe5a6a4316..561f1bb8ad 100644 --- a/beacon_chain/validator_client/duties_service.nim +++ b/beacon_chain/validator_client/duties_service.nim @@ -19,7 +19,8 @@ logScope: service = ServiceName type DutiesServiceLoop* = enum AttesterLoop, ProposerLoop, IndicesLoop, SyncCommitteeLoop, - ProposerPreparationLoop, ValidatorRegisterLoop, DynamicValidatorsLoop + ProposerPreparationLoop, ValidatorRegisterLoop, DynamicValidatorsLoop, + SlashPruningLoop chronicles.formatIt(DutiesServiceLoop): case it @@ -30,6 +31,7 @@ chronicles.formatIt(DutiesServiceLoop): of ProposerPreparationLoop: "proposer_prepare_loop" of ValidatorRegisterLoop: "validator_register_loop" of DynamicValidatorsLoop: "dynamic_validators_loop" + of SlashPruningLoop: "slashing_pruning_loop" proc checkDuty(duty: RestAttesterDuty): bool = (duty.committee_length <= MAX_VALIDATORS_PER_COMMITTEE) and @@ -677,6 +679,70 @@ proc syncCommitteeDutiesLoop(service: DutiesServiceRef) {.async.} = # Spawning new attestation duties task. service.pollingSyncDutiesTask = service.pollForSyncCommitteeDuties() +proc getNextEpochMiddleSlot(vc: ValidatorClientRef): Slot = + let + middleSlot = Slot(SLOTS_PER_EPOCH div 2) + currentSlot = vc.beaconClock.now().slotOrZero() + slotInEpoch = currentSlot.since_epoch_start() + + if slotInEpoch >= middleSlot: + (currentSlot.epoch + 1'u64).start_slot() + uint64(middleSlot) + else: + currentSlot + (uint64(middleSlot) - uint64(slotInEpoch)) + +proc pruneSlashingDatabase(service: DutiesServiceRef) {.async.} = + let + vc = service.client + currentSlot = vc.beaconClock.now().slotOrZero() + startTime = Moment.now() + blockHeader = + try: + await vc.getFinalizedBlockHeader() + except CancelledError as exc: + debug "Finalized block header request was interrupted", + slot = currentSlot + raise exc + except CatchableError as exc: + error "Unexpected error occured while requesting " & + "finalized block header", slot = currentSlot, + err_name = exc.name, err_msg = exc.msg + Opt.none(GetBlockHeaderResponse) + checkpointTime = Moment.now() + if blockHeader.isSome(): + let epoch = blockHeader.get().data.header.message.slot.epoch + vc.finalizedEpoch = Opt.some(epoch) + if service.lastSlashingEpoch.get(FAR_FUTURE_EPOCH) != epoch: + vc.attachedValidators[] + .slashingProtection + .pruneAfterFinalization(epoch) + service.lastSlashingEpoch = Opt.some(epoch) + let finishTime = Moment.now() + debug "Slashing database has been pruned", slot = currentSlot, + epoch = currentSlot.epoch(), + finalized_epoch = epoch, + elapsed_time = (finishTime - startTime), + pruning_time = (finishTime - checkpointTime) + +proc slashingDatabasePruningLoop(service: DutiesServiceRef) {.async.} = + let vc = service.client + debug "Slashing database pruning loop is waiting for initialization" + await allFutures( + vc.preGenesisEvent.wait(), + vc.indicesAvailable.wait(), + vc.forksAvailable.wait() + ) + doAssert(len(vc.forks) > 0, "Fork schedule must not be empty at this point") + while true: + let slot = await vc.checkedWaitForSlot(vc.getNextEpochMiddleSlot(), + aggregateSlotOffset, false) + if slot.isNone(): + continue + + if not(isNil(service.pruneSlashingDatabaseTask)) and + not(service.pruneSlashingDatabaseTask.finished()): + await cancelAndWait(service.pruneSlashingDatabaseTask) + service.pruneSlashingDatabaseTask = service.pruneSlashingDatabase() + template checkAndRestart(serviceLoop: DutiesServiceLoop, future: Future[void], body: untyped): untyped = if future.finished(): @@ -715,6 +781,7 @@ proc mainLoop(service: DutiesServiceRef) {.async.} = else: debug "Dynamic validators update loop disabled" @[] + slashPruningFut = service.slashingDatabasePruningLoop() web3SignerUrls = vc.config.web3SignerUrls while true: @@ -729,6 +796,7 @@ proc mainLoop(service: DutiesServiceRef) {.async.} = FutureBase(indicesFut), FutureBase(syncFut), FutureBase(prepareFut), + FutureBase(slashPruningFut) ] for fut in dynamicFuts: futures.add fut @@ -749,6 +817,8 @@ proc mainLoop(service: DutiesServiceRef) {.async.} = service.dynamicValidatorsLoop( web3SignerUrls[i], vc.config.web3signerUpdateInterval)) + checkAndRestart(SlashPruningLoop, slashPruningFut, + service.slashingDatabasePruningLoop()) false except CancelledError: debug "Service interrupted" @@ -774,6 +844,9 @@ proc mainLoop(service: DutiesServiceRef) {.async.} = if not(isNil(service.pollingSyncDutiesTask)) and not(service.pollingSyncDutiesTask.finished()): pending.add(service.pollingSyncDutiesTask.cancelAndWait()) + if not(isNil(service.pruneSlashingDatabaseTask)) and + not(service.pruneSlashingDatabaseTask.finished()): + pending.add(service.pruneSlashingDatabaseTask.cancelAndWait()) await allFutures(pending) true except CatchableError as exc: From 8a98de6bd098d30c1e2218172221cfb185cd44db Mon Sep 17 00:00:00 2001 From: tersec Date: Tue, 7 Nov 2023 05:12:16 +0100 Subject: [PATCH 35/35] bump holesky to remove large, extraneous JSON file (#5574) --- vendor/holesky | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/holesky b/vendor/holesky index ea39b90062..bd43fac068 160000 --- a/vendor/holesky +++ b/vendor/holesky @@ -1 +1 @@ -Subproject commit ea39b9006210848e13f28d92e12a30548cecd41d +Subproject commit bd43fac06847124d4dadcc994401a4993b5fe9d1