From 706824cf959df3aaa5aef543d573aaae00c411a3 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Thu, 8 Jun 2023 20:25:03 +0300 Subject: [PATCH 1/6] Drop need to declare spec feature files --- setup.py | 121 ++++++++++++++++++++++++++----------------------------- 1 file changed, 57 insertions(+), 64 deletions(-) diff --git a/setup.py b/setup.py index f4d0c87197..f9b2438d48 100644 --- a/setup.py +++ b/setup.py @@ -50,6 +50,25 @@ def installPackage(package: str): EIP6110 = 'eip6110' WHISK = 'whisk' +PREVIOUS_FORK_OF = { + PHASE0: None, + ALTAIR: PHASE0, + BELLATRIX: ALTAIR, + CAPELLA: BELLATRIX, + DENEB: CAPELLA, + EIP6110: DENEB, + WHISK: CAPELLA, +} + +ALL_FORKS = list(PREVIOUS_FORK_OF.keys()) + +IGNORE_SPEC_FILES = [ + "specs/phase0/deposit-contract.md" +] + +EXTRA_SPEC_FILES = { + BELLATRIX: "sync/optimistic.md" +} # The helper functions that are used when defining constants CONSTANT_DEP_SUNDRY_CONSTANTS_FUNCTIONS = ''' @@ -96,6 +115,30 @@ class SpecObject(NamedTuple): dataclasses: Dict[str, str] +def is_post_fork(a, b) -> bool: + """ + Returns true if fork a is after b, or if a == b + """ + if a == b: + return True + + prev_fork = PREVIOUS_FORK_OF[a] + if prev_fork == b: + return True + elif prev_fork == None: + return False + else: + return is_post_fork(prev_fork, b) + +def get_fork_directory(fork): + dir1 = f'specs/{fork}' + if os.path.exists(dir1): + return dir1 + dir2 = f'specs/_features/{fork}' + if os.path.exists(dir2): + return dir2 + raise FileNotFoundError(f"No directory found for fork: {fork}") + def _get_name_from_heading(heading: Heading) -> Optional[str]: last_child = heading.children[-1] if isinstance(last_child, CodeSpan): @@ -1068,70 +1111,20 @@ def finalize_options(self): if len(self.md_doc_paths) == 0: print("no paths were specified, using default markdown file paths for pyspec" " build (spec fork: %s)" % self.spec_fork) - if self.spec_fork in (PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, EIP6110, WHISK): - self.md_doc_paths = """ - specs/phase0/beacon-chain.md - specs/phase0/fork-choice.md - specs/phase0/validator.md - specs/phase0/weak-subjectivity.md - specs/phase0/p2p-interface.md - """ - if self.spec_fork in (ALTAIR, BELLATRIX, CAPELLA, DENEB, EIP6110, WHISK): - self.md_doc_paths += """ - specs/altair/light-client/full-node.md - specs/altair/light-client/light-client.md - specs/altair/light-client/p2p-interface.md - specs/altair/light-client/sync-protocol.md - specs/altair/beacon-chain.md - specs/altair/bls.md - specs/altair/fork.md - specs/altair/validator.md - specs/altair/p2p-interface.md - """ - if self.spec_fork in (BELLATRIX, CAPELLA, DENEB, EIP6110, WHISK): - self.md_doc_paths += """ - specs/bellatrix/beacon-chain.md - specs/bellatrix/fork.md - specs/bellatrix/fork-choice.md - specs/bellatrix/validator.md - specs/bellatrix/p2p-interface.md - sync/optimistic.md - """ - if self.spec_fork in (CAPELLA, DENEB, EIP6110, WHISK): - self.md_doc_paths += """ - specs/capella/light-client/fork.md - specs/capella/light-client/full-node.md - specs/capella/light-client/p2p-interface.md - specs/capella/light-client/sync-protocol.md - specs/capella/beacon-chain.md - specs/capella/fork.md - specs/capella/fork-choice.md - specs/capella/validator.md - specs/capella/p2p-interface.md - """ - if self.spec_fork in (DENEB, EIP6110): - self.md_doc_paths += """ - specs/deneb/light-client/fork.md - specs/deneb/light-client/full-node.md - specs/deneb/light-client/p2p-interface.md - specs/deneb/light-client/sync-protocol.md - specs/deneb/beacon-chain.md - specs/deneb/fork.md - specs/deneb/fork-choice.md - specs/deneb/polynomial-commitments.md - specs/deneb/p2p-interface.md - specs/deneb/validator.md - """ - if self.spec_fork == EIP6110: - self.md_doc_paths += """ - specs/_features/eip6110/beacon-chain.md - specs/_features/eip6110/fork.md - """ - if self.spec_fork == WHISK: - self.md_doc_paths += """ - specs/_features/whisk/beacon-chain.md - specs/_features/whisk/fork.md - """ + self.md_doc_paths = "" + + for fork in ALL_FORKS: + if is_post_fork(self.spec_fork, fork): + # Append all files in fork directory recursively + for root, dirs, files in os.walk(get_fork_directory(fork)): + for filename in files: + filepath = os.path.join(root, filename) + if filepath.endswith('.md') and filepath not in IGNORE_SPEC_FILES: + self.md_doc_paths += filepath + "\n" + # Append extra files if any + if fork in EXTRA_SPEC_FILES: + self.md_doc_paths += EXTRA_SPEC_FILES[fork] + "\n" + if len(self.md_doc_paths) == 0: raise Exception('no markdown files specified, and spec fork "%s" is unknown', self.spec_fork) From cfd44dac56996b8195434277392686fc319b70d2 Mon Sep 17 00:00:00 2001 From: William Dowling Date: Sun, 11 Jun 2023 15:07:39 +0200 Subject: [PATCH 2/6] Update lru-dict package to latest stable release. lru-dict build fails when building under clang version 16 (https://github.com/amitdev/lru-dict/commit/6badf6376d12a2e0498f67bbc7d5c0332b96a4d7). This bumps the version to the latest stable release. Tested on MacOS M2 and Ubuntu Jammy x86-64. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f4d0c87197..adf24ba363 100644 --- a/setup.py +++ b/setup.py @@ -1284,7 +1284,7 @@ def run(self): "remerkleable==0.1.27", "trie==2.0.2", RUAMEL_YAML_VERSION, - "lru-dict==1.1.8", + "lru-dict==1.2.0", MARKO_VERSION, "py_arkworks_bls12381==0.3.4", "curdleproofs @ git+https://github.com/nalinbhardwaj/curdleproofs.pie@master#egg=curdleproofs&subdirectory=curdleproofs", From 7cf55c138ceb4bcbad889c362b9dce6d1c130afb Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Mon, 12 Jun 2023 15:34:05 +0300 Subject: [PATCH 3/6] Lock nalinbhardwaj/curdleproofs.pie version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f4d0c87197..f6d69cc13b 100644 --- a/setup.py +++ b/setup.py @@ -1287,6 +1287,6 @@ def run(self): "lru-dict==1.1.8", MARKO_VERSION, "py_arkworks_bls12381==0.3.4", - "curdleproofs @ git+https://github.com/nalinbhardwaj/curdleproofs.pie@master#egg=curdleproofs&subdirectory=curdleproofs", + "curdleproofs @ git+https://github.com/nalinbhardwaj/curdleproofs.pie@805d06785b6ff35fde7148762277dd1ae678beeb#egg=curdleproofs&subdirectory=curdleproofs", ] ) From 512d2ca516d3aa6c3b65a831ff89c2bacaa6f9da Mon Sep 17 00:00:00 2001 From: gajinder Date: Mon, 12 Jun 2023 20:24:22 +0530 Subject: [PATCH 4/6] Update blob side car subnet count to 6 in line with max blobs limit update --- specs/deneb/validator.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/deneb/validator.md b/specs/deneb/validator.md index b53c0a042d..ec9c547849 100644 --- a/specs/deneb/validator.md +++ b/specs/deneb/validator.md @@ -46,7 +46,7 @@ Please see related Beacon Chain doc before continuing and use them as a referenc | Name | Value | Unit | | - | - | :-: | -| `BLOB_SIDECAR_SUBNET_COUNT` | `4` | *[New in Deneb:EIP4844]* The number of blob sidecar subnets used in the gossipsub protocol. | +| `BLOB_SIDECAR_SUBNET_COUNT` | `6` | *[New in Deneb:EIP4844]* The number of blob sidecar subnets used in the gossipsub protocol. | ## Helpers From 210c4827bc6824c02c7c0cfcc7aad26294767560 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 12 Jun 2023 23:57:21 +0800 Subject: [PATCH 5/6] Move `BLOB_SIDECAR_SUBNET_COUNT` to networking config --- configs/mainnet.yaml | 2 ++ configs/minimal.yaml | 2 ++ specs/deneb/p2p-interface.md | 7 +++++-- specs/deneb/validator.md | 10 ---------- .../test/deneb/unittests/test_config_invariants.py | 2 ++ 5 files changed, 11 insertions(+), 12 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index fe405cb38a..365bc11367 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -132,3 +132,5 @@ MAX_REQUEST_BLOCKS_DENEB: 128 MAX_REQUEST_BLOB_SIDECARS: 768 # `2**12` (= 4096 epochs, ~18 days) MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 4096 +# `6` +BLOB_SIDECAR_SUBNET_COUNT: 6 diff --git a/configs/minimal.yaml b/configs/minimal.yaml index dcc834963e..b22a7165e1 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -133,3 +133,5 @@ MAX_REQUEST_BLOCKS_DENEB: 128 MAX_REQUEST_BLOB_SIDECARS: 768 # `2**12` (= 4096 epochs, ~18 days) MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 4096 +# `6` +BLOB_SIDECAR_SUBNET_COUNT: 6 diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index 32e0ee9bf5..809e405a66 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -40,11 +40,14 @@ The specification of these changes continues in the same format as the network s ### Configuration +*[New in Deneb:EIP4844]* + | Name | Value | Description | |------------------------------------------|-----------------------------------|---------------------------------------------------------------------| | `MAX_REQUEST_BLOCKS_DENEB` | `2**7` (= 128) | Maximum number of blocks in a single request | -| `MAX_REQUEST_BLOB_SIDECARS` | `MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK` | Maximum number of blob sidecars in a single request | -| `MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS` | `2**12` (= 4096 epochs, ~18 days) | The minimum epoch range over which a node must serve blob sidecars | +| `MAX_REQUEST_BLOB_SIDECARS` | `MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK` | Maximum number of blob sidecars in a single request | +| `MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS` | `2**12` (= 4096 epochs, ~18 days) | The minimum epoch range over which a node must serve blob sidecars | +| `BLOB_SIDECAR_SUBNET_COUNT` | `6` | The number of blob sidecar subnets used in the gossipsub protocol. | ### Containers diff --git a/specs/deneb/validator.md b/specs/deneb/validator.md index ec9c547849..3157ccf218 100644 --- a/specs/deneb/validator.md +++ b/specs/deneb/validator.md @@ -10,8 +10,6 @@ - [Introduction](#introduction) - [Prerequisites](#prerequisites) -- [Constants](#constants) - - [Misc](#misc) - [Helpers](#helpers) - [`BlobsBundle`](#blobsbundle) - [Modified `GetPayloadResponse`](#modified-getpayloadresponse) @@ -40,14 +38,6 @@ All behaviors and definitions defined in this document, and documents it extends All terminology, constants, functions, and protocol mechanics defined in the updated [Beacon Chain doc of Deneb](./beacon-chain.md) are requisite for this document and used throughout. Please see related Beacon Chain doc before continuing and use them as a reference throughout. -## Constants - -### Misc - -| Name | Value | Unit | -| - | - | :-: | -| `BLOB_SIDECAR_SUBNET_COUNT` | `6` | *[New in Deneb:EIP4844]* The number of blob sidecar subnets used in the gossipsub protocol. | - ## Helpers ### `BlobsBundle` diff --git a/tests/core/pyspec/eth2spec/test/deneb/unittests/test_config_invariants.py b/tests/core/pyspec/eth2spec/test/deneb/unittests/test_config_invariants.py index 75955ed161..087bd63c60 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/unittests/test_config_invariants.py +++ b/tests/core/pyspec/eth2spec/test/deneb/unittests/test_config_invariants.py @@ -18,3 +18,5 @@ def test_length(spec): def test_networking(spec): assert spec.MAX_BLOBS_PER_BLOCK < spec.MAX_BLOB_COMMITMENTS_PER_BLOCK assert spec.config.MAX_REQUEST_BLOB_SIDECARS == spec.config.MAX_REQUEST_BLOCKS_DENEB * spec.MAX_BLOBS_PER_BLOCK + # Start with the same size, but `BLOB_SIDECAR_SUBNET_COUNT` could potentially increase later. + assert spec.config.BLOB_SIDECAR_SUBNET_COUNT == spec.MAX_BLOBS_PER_BLOCK From 87f77a3a6d5d5cb240e9961611514606b6d44f2f Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 13 Jun 2023 01:42:18 +0800 Subject: [PATCH 6/6] bump VERSION.txt --- tests/core/pyspec/eth2spec/VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/VERSION.txt b/tests/core/pyspec/eth2spec/VERSION.txt index 54dd11e46b..45eafc27d4 100644 --- a/tests/core/pyspec/eth2spec/VERSION.txt +++ b/tests/core/pyspec/eth2spec/VERSION.txt @@ -1 +1 @@ -1.4.0-alpha.2 +1.4.0-alpha.3