ethereum · djrtwo · Apr 18, 2023 · Feb 16, 2023 · Mar 24, 2023 · Mar 25, 2023
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -0,0 +1,24 @@
+
+name: Publish docs
+on:
+  push:
+    branches:
+      - master 
+permissions:
+  contents: write
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Build docs
+        run: make copy_docs
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.x
+      - uses: actions/cache@v2
+        with:
+          key: ${{ github.ref }}
+          path: .cache
+      - run: pip install -e .[docs]
+      - run: mkdocs gh-deploy --force
diff --git a/.gitignore b/.gitignore
@@ -35,3 +35,11 @@ tests/core/pyspec/eth2spec/test_results.xml
 
 # TOC tool outputs temporary files
 *.tmp
+
+# docs reader build
+docs/specs
+docs/sync
+docs/ssz
+docs/fork_choice
+docs/README.md
+site
diff --git a/Makefile b/Makefile
@@ -19,6 +19,11 @@ GENERATORS = $(sort $(dir $(wildcard $(GENERATOR_DIR)/*/.)))
 # Map this list of generator paths to "gen_{generator name}" entries
 GENERATOR_TARGETS = $(patsubst $(GENERATOR_DIR)/%/, gen_%, $(GENERATORS))
 GENERATOR_VENVS = $(patsubst $(GENERATOR_DIR)/%, $(GENERATOR_DIR)/%venv, $(GENERATORS))
+# Documents
+DOCS_DIR = ./docs
+SSZ_DIR = ./ssz
+SYNC_DIR = ./sync
+FORK_CHOICE_DIR = ./fork_choice
 
 # To check generator matching:
 #$(info $$GENERATOR_TARGETS is [${GENERATOR_TARGETS}])
@@ -214,3 +219,23 @@ detect_generator_incomplete: $(TEST_VECTOR_DIR)
 
 detect_generator_error_log: $(TEST_VECTOR_DIR)
 	[ -f $(GENERATOR_ERROR_LOG_FILE) ] && echo "[ERROR] $(GENERATOR_ERROR_LOG_FILE) file exists" || echo "[PASSED] error log file does not exist"
+
+
+# For docs reader
+install_docs:
+	python3 -m venv venv; . venv/bin/activate; python3 -m pip install -e .[docs];
+
+copy_docs:
+	cp -r $(SPEC_DIR) $(DOCS_DIR);
+	cp -r $(SYNC_DIR) $(DOCS_DIR);
+	cp -r $(SSZ_DIR) $(DOCS_DIR);
+	cp -r $(FORK_CHOICE_DIR) $(DOCS_DIR);
+	cp $(CURRENT_DIR)/README.md $(DOCS_DIR)/README.md
+
+build_docs: copy_docs
+	. venv/bin/activate;
+	mkdocs build
+
+serve_docs:
+	. venv/bin/activate; 
+	mkdocs serve
diff --git a/README.md b/README.md
@@ -1,6 +1,6 @@
 # Ethereum Proof-of-Stake Consensus Specifications
 
-[![Join the chat at https://discord.gg/qGpsxSA](https://img.shields.io/badge/chat-on%20discord-blue.svg)](https://discord.gg/qGpsxSA) [![Join the chat at https://gitter.im/ethereum/sharding](https://badges.gitter.im/ethereum/sharding.svg)](https://gitter.im/ethereum/sharding?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
+[![Join the chat at https://discord.gg/qGpsxSA](https://img.shields.io/badge/chat-on%20discord-blue.svg)](https://discord.gg/qGpsxSA)
 
 To learn more about proof-of-stake and sharding, see the [PoS documentation](https://ethereum.org/en/developers/docs/consensus-mechanisms/pos/), [sharding documentation](https://ethereum.org/en/upgrades/sharding/) and the [research compendium](https://notes.ethereum.org/s/H1PGqDhpm).
 
@@ -65,6 +65,10 @@ Documentation on the different components used during spec writing can be found
 * [YAML Test Generators](tests/generators/README.md)
 * [Executable Python Spec, with Py-tests](tests/core/pyspec/README.md)
 
+## Online viewer of the latest release (latest `master` branch)
+
+[Ethereum Consensus Specs](https://ethereum.github.io/consensus-specs/)
+
 ## Consensus spec tests
 
 Conformance tests built from the executable python spec are available in the [Ethereum Proof-of-Stake Consensus Spec Tests](https://github.com/ethereum/consensus-spec-tests) repo. Compressed tarballs are available in [releases](https://github.com/ethereum/consensus-spec-tests/releases).
diff --git a/docs/.pages b/docs/.pages
@@ -0,0 +1,5 @@
+nav:
+    - Home:
+        - README.md
+    - specs
+    - ...
diff --git a/docs/README.md b/docs/README.md
@@ -0,0 +1,70 @@
+# Ethereum Proof-of-Stake Consensus Specifications
+
+[![Join the chat at https://discord.gg/qGpsxSA](https://img.shields.io/badge/chat-on%20discord-blue.svg)](https://discord.gg/qGpsxSA) [![Join the chat at https://gitter.im/ethereum/sharding](https://badges.gitter.im/ethereum/sharding.svg)](https://gitter.im/ethereum/sharding?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
+
+To learn more about proof-of-stake and sharding, see the [PoS documentation](https://ethereum.org/en/developers/docs/consensus-mechanisms/pos/), [sharding documentation](https://ethereum.org/en/upgrades/sharding/) and the [research compendium](https://notes.ethereum.org/s/H1PGqDhpm).
+
+This repository hosts the current Ethereum proof-of-stake specifications. Discussions about design rationale and proposed changes can be brought up and discussed as issues. Solidified, agreed-upon changes to the spec can be made through pull requests.
+
+## Specs
+
+[![GitHub release](https://img.shields.io/github/v/release/ethereum/eth2.0-specs)](https://github.com/ethereum/eth2.0-specs/releases/) [![PyPI version](https://badge.fury.io/py/eth2spec.svg)](https://badge.fury.io/py/eth2spec)
+
+Core specifications for Ethereum proof-of-stake clients can be found in [specs](specs/). These are divided into features.
+Features are researched and developed in parallel, and then consolidated into sequential upgrades when ready.
+
+### Stable Specifications
+
+| Seq. | Code Name | Fork Epoch | Specs |
+| - | - | - | - |
+| 0 | **Phase0** |`0` | <ul><li>Core</li><ul><li>[The beacon chain](specs/phase0/beacon-chain.md)</li><li>[Deposit contract](specs/phase0/deposit-contract.md)</li><li>[Beacon chain fork choice](specs/phase0/fork-choice.md)</li></ul><li>Additions</li><ul><li>[Honest validator guide](specs/phase0/validator.md)</li><li>[P2P networking](specs/phase0/p2p-interface.md)</li><li>[Weak subjectivity](specs/phase0/weak-subjectivity.md)</li></ul></ul> |
+| 1 |  **Altair** | `74240` | <ul><li>Core</li><ul><li>[Beacon chain changes](specs/altair/beacon-chain.md)</li><li>[Altair fork](specs/altair/fork.md)</li></ul><li>Additions</li><ul><li>[Light client sync protocol](specs/altair/light-client/sync-protocol.md) ([full node](specs/altair/light-client/full-node.md), [light client](specs/altair/light-client/light-client.md), [networking](specs/altair/light-client/p2p-interface.md))</li><li>[Honest validator guide changes](specs/altair/validator.md)</li><li>[P2P networking](specs/altair/p2p-interface.md)</li></ul></ul> |
+| 2 | **Bellatrix** <br/> (["The Merge"](https://ethereum.org/en/upgrades/merge/)) | `144896` | <ul><li>Core</li><ul><li>[Beacon Chain changes](specs/bellatrix/beacon-chain.md)</li><li>[Bellatrix fork](specs/bellatrix/fork.md)</li><li>[Fork choice changes](specs/bellatrix/fork-choice.md)</li></ul><li>Additions</li><ul><li>[Honest validator guide changes](specs/bellatrix/validator.md)</li><li>[P2P networking](specs/bellatrix/p2p-interface.md)</li></ul></ul> |
+| 3 | **Capella** | `194048` | <ul><li>Core</li><ul><li>[Beacon chain changes](specs/capella/beacon-chain.md)</li><li>[Capella fork](specs/capella/fork.md)</li></ul><li>Additions</li><ul><li>[Light client sync protocol changes](specs/capella/light-client/sync-protocol.md) ([fork](specs/capella/light-client/fork.md), [full node](specs/capella/light-client/full-node.md), [networking](specs/capella/light-client/p2p-interface.md))</li></ul><ul><li>[Validator additions](specs/capella/validator.md)</li><li>[P2P networking](specs/capella/p2p-interface.md)</li></ul></ul> |
+
+### In-development Specifications
+| Code Name or Topic | Specs | Notes |
+| - | - | - |
+| Deneb (tentative) | <ul><li>Core</li><ul><li>[Beacon Chain changes](specs/deneb/beacon-chain.md)</li><li>[Deneb fork](specs/deneb/fork.md)</li><li>[Polynomial commitments](specs/deneb/polynomial-commitments.md)</li><li>[Fork choice changes](specs/deneb/fork-choice.md)</li></ul><li>Additions</li><ul><li>[Light client sync protocol changes](specs/deneb/light-client/sync-protocol.md) ([fork](specs/deneb/light-client/fork.md), [full node](specs/deneb/light-client/full-node.md), [networking](specs/deneb/light-client/p2p-interface.md))</li></ul><ul><li>[Honest validator guide changes](specs/deneb/validator.md)</li><li>[P2P networking](specs/deneb/p2p-interface.md)</li></ul></ul> |
+| Sharding (outdated) | <ul><li>Core</li><ul><li>[Beacon Chain changes](specs/_features/sharding/beacon-chain.md)</li></ul><li>Additions</li><ul><li>[P2P networking](specs/_features/sharding/p2p-interface.md)</li></ul></ul> |
+| Custody Game (outdated) | <ul><li>Core</li><ul><li>[Beacon Chain changes](specs/_features/custody_game/beacon-chain.md)</li></ul><li>Additions</li><ul><li>[Honest validator guide changes](specs/_features/custody_game/validator.md)</li></ul></ul> | Dependent on sharding |
+| Data Availability Sampling (outdated) | <ul><li>Core</li><ul><li>[Core types and functions](specs/_features/das/das-core.md)</li><li>[Fork choice changes](specs/_features/das/fork-choice.md)</li></ul><li>Additions</li><ul><li>[P2P Networking](specs/_features/das/p2p-interface.md)</li><li>[Sampling process](specs/_features/das/sampling.md)</li></ul></ul> | <ul><li> Dependent on sharding</li><li>[Technical explainer](https://hackmd.io/@HWeNw8hNRimMm2m2GH56Cw/B1YJPGkpD)</li></ul> |
+| EIP-6110 | <ul><li>Core</li><ul><li>[Beacon Chain changes](specs/_features/eip6110//beacon-chain.md)</li><li>[EIP-6110 fork](specs/_features/eip6110/fork.md)</li></ul><li>Additions</li><ul><li>[Honest validator guide changes](specs/_features/eip6110/validator.md)</li></ul></ul> |
+
+### Accompanying documents can be found in [specs](specs) and include:
+
+* [SimpleSerialize (SSZ) spec](ssz/simple-serialize.md)
+* [Merkle proof formats](ssz/merkle-proofs.md)
+* [General test format](tests/formats/README.md)
+
+## Additional specifications for client implementers
+
+Additional specifications and standards outside of requisite client functionality can be found in the following repos:
+
+* [Beacon APIs](https://github.com/ethereum/beacon-apis)
+* [Beacon Metrics](https://github.com/ethereum/beacon-metrics/)
+
+## Design goals
+
+The following are the broad design goals for the Ethereum proof-of-stake consensus specifications:
+* to minimize complexity, even at the cost of some losses in efficiency
+* to remain live through major network partitions and when very large portions of nodes go offline
+* to select all components such that they are either quantum secure or can be easily swapped out for quantum secure counterparts when available
+* to utilize crypto and design techniques that allow for a large participation of validators in total and per unit time
+* to allow for a typical consumer laptop with `O(C)` resources to process/validate `O(1)` shards (including any system level validation such as the beacon chain)
+
+## Useful external resources
+
+* [Design Rationale](https://notes.ethereum.org/s/rkhCgQteN#)
+* [Phase 0 Onboarding Document](https://notes.ethereum.org/s/Bkn3zpwxB)
+* [Combining GHOST and Casper paper](https://arxiv.org/abs/2003.03052)
+
+## For spec contributors
+
+Documentation on the different components used during spec writing can be found here:
+* [YAML Test Generators](tests/generators/README.md)
+* [Executable Python Spec, with Py-tests](tests/core/pyspec/README.md)
+
+## Consensus spec tests
+
+Conformance tests built from the executable python spec are available in the [Ethereum Proof-of-Stake Consensus Spec Tests](https://github.com/ethereum/consensus-spec-tests) repo. Compressed tarballs are available in [releases](https://github.com/ethereum/consensus-spec-tests/releases).
diff --git a/docs/docs/new-feature.md b/docs/docs/new-feature.md
@@ -0,0 +1,163 @@
+# How to add a new feature proposal in consensus-specs
+
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+## Table of Contents
+
+- [A. Make it executable for linter checks](#a-make-it-executable-for-linter-checks)
+  - [1. Create a folder under `./specs/_features`](#1-create-a-folder-under-specs_features)
+  - [2. Choose the "previous fork" to extend: usually, use the scheduled or the latest mainnet fork version.](#2-choose-the-previous-fork-to-extend-usually-use-the-scheduled-or-the-latest-mainnet-fork-version)
+  - [3. Write down your proposed `beacon-chain.md` change](#3-write-down-your-proposed-beacon-chainmd-change)
+  - [4. Add `fork.md`](#4-add-forkmd)
+  - [5. Make it executable](#5-make-it-executable)
+- [B: Make it executable for pytest and test generator](#b-make-it-executable-for-pytest-and-test-generator)
+  - [1. Add `light-client/*` docs if you updated the content of `BeaconBlock`](#1-add-light-client-docs-if-you-updated-the-content-of-beaconblock)
+  - [2. Add the mainnet and minimal presets and update the configs](#2-add-the-mainnet-and-minimal-presets-and-update-the-configs)
+  - [3. Update `context.py`](#3-update-contextpy)
+  - [4. Update `constants.py`](#4-update-constantspy)
+  - [5. Update `genesis.py`:](#5-update-genesispy)
+  - [6. To add fork transition tests, update fork_transition.py](#6-to-add-fork-transition-tests-update-fork_transitionpy)
+  - [7. Update CI configurations](#7-update-ci-configurations)
+- [Others](#others)
+  - [Bonus](#bonus)
+  - [Need help?](#need-help)
+
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
+
+
+## A. Make it executable for linter checks
+
+### 1. Create a folder under `./specs/_features`
+
+For example, if it's an `EIP-9999` CL spec, you can create a `./specs/_features/eip9999` folder.
+
+### 2. Choose the "previous fork" to extend: usually, use the scheduled or the latest mainnet fork version.
+
+For example, if the latest fork is Capella, use `./specs/capella` content as your "previous fork".
+
+### 3. Write down your proposed `beacon-chain.md` change
+- You can either use [Beacon Chain Spec Template](./templates/beacon-chain-template.md), or make a copy of the latest fork content and then edit it.
+- Tips:
+    - We use [`doctoc`](https://www.npmjs.com/package/doctoc) tool to generate the table of content.
+        ```
+        cd consensus-specs
+        doctoc specs
+        ```
+    - The differences between "Constants", "Configurations", and "Presets":
+        - Constants: The constant that should never be changed.
+        - Configurations: The settings that we may change for different networks.
+        - Presets: The settings that we may change for testing.
+    - Readability and simplicity are more important than efficiency and optimization.
+        - Use simple Python rather than the fancy Python dark magic.
+
+### 4. Add `fork.md`
+You can refer to the previous fork's `fork.md` file.
+### 5. Make it executable
+- Update [`constants.py`](https://github.com/ethereum/consensus-specs/blob/dev/tests/core/pyspec/eth2spec/test/helpers/constants.py) with the new feature name.
+- Update [`setup.py`](https://github.com/ethereum/consensus-specs/blob/dev/setup.py):
+    - Add a new `SpecBuilder` with the new feature name constant. e.g., `EIP9999SpecBuilder`
+    - Add the new `SpecBuilder` to `spec_builders` list.
+    - Add the path of the new markdown files in `finalize_options` function.
+
+## B: Make it executable for pytest and test generator
+
+### 1. Add `light-client/*` docs if you updated the content of `BeaconBlock`
+- You can refer to the previous fork's `light-client/*` file.
+- Add the path of the new markdown files in `setup.py`'s `finalize_options` function.
+
+### 2. Add the mainnet and minimal presets and update the configs
+- Add presets: `presets/mainnet/<new-feature-name>.yaml` and `presets/minimal/<new-feature-name>.yaml`
+- Update configs: `configs/mainnet.yaml` and `configs/minimal.yaml`
+
+### 3. Update [`context.py`](https://github.com/ethereum/consensus-specs/blob/dev/tests/core/pyspec/eth2spec/test/context.py)
+- Update `spec_targets` by adding `<NEW_FEATURE>`
+
+```python
+from eth2spec.eip9999 import mainnet as spec_eip9999_mainnet, minimal as spec_eip9999_minimal
+
+...
+
+spec_targets: Dict[PresetBaseName, Dict[SpecForkName, Spec]] = {
+    MINIMAL: {
+        ...
+        EIP9999: spec_eip9999_minimal,
+    },
+    MAINNET: {
+        ...
+        EIP9999: spec_eip9999_mainnet
+    },
+}
+```
+
+### 4. Update [`constants.py`](https://github.com/ethereum/consensus-specs/blob/dev/tests/core/pyspec/eth2spec/test/helpers/constants.py)
+- Add `<NEW_FEATURE>` to `ALL_PHASES` and `TESTGEN_FORKS`
+
+### 5. Update [`genesis.py`](https://github.com/ethereum/consensus-specs/blob/dev/tests/core/pyspec/eth2spec/test/helpers/genesis.py):
+
+We use `create_genesis_state` to create the default `state` in tests.
+
+- Update `create_genesis_state` by adding `fork_version` setting:
+
+```python
+def create_genesis_state(spec, validator_balances, activation_threshold):
+    ...
+    if spec.fork == ALTAIR:
+        current_version = spec.config.ALTAIR_FORK_VERSION
+    ...
+    elif spec.fork == EIP9999:
+        # Add the previous fork version of given fork
+        previous_version = spec.config.<PREVIOUS_FORK_VERSION>
+        current_version = spec.config.EIP9999_FORK_VERSION
+```
+
+- If the given feature changes `BeaconState` fields, you have to set the initial values by adding:
+
+```python
+def create_genesis_state(spec, validator_balances, activation_threshold):
+    ...
+    if is_post_eip9999(spec):
+        state.<NEW_FIELD> = <value>
+
+    return state
+```
+
+- If the given feature changes `ExecutionPayload` fields, you have to set the initial values by updating `get_sample_genesis_execution_payload_header` helper.
+
+### 6. To add fork transition tests, update [fork_transition.py](https://github.com/ethereum/consensus-specs/blob/dev/tests/core/pyspec/eth2spec/test/helpers/fork_transition.py)
+
+```python
+def do_fork(state, spec, post_spec, fork_epoch, with_block=True, sync_aggregate=None, operation_dict=None):
+    ...
+
+    if post_spec.fork == ALTAIR:
+        state = post_spec.upgrade_to_altair(state)
+    ...
+    elif post_spec.fork == EIP9999:
+        state = post_spec.upgrade_to_eip9999(state)
+
+    ...
+
+    if post_spec.fork == ALTAIR:
+        assert state.fork.previous_version == post_spec.config.GENESIS_FORK_VERSION
+        assert state.fork.current_version == post_spec.config.ALTAIR_FORK_VERSION
+    ...
+    elif post_spec.fork == EIP9999:
+        assert state.fork.previous_version == post_spec.config.<PREVIOUS_FORK_VERSION>
+        assert state.fork.current_version == post_spec.config.EIP9999_FORK_VERSION
+
+    ...
+```
+
+### 7. Update CI configurations
+- Update [GitHub Actions config](https://github.com/ethereum/consensus-specs/blob/dev/.github/workflows/run-tests.yml)
+    - Update `pyspec-tests.strategy.matrix.version` list by adding new feature to it
+- Update [CircleCI config](https://github.com/ethereum/consensus-specs/blob/dev/.circleci/config.yml)
+    - Add new job to the `workflows.test_spec.jobs`
+
+## Others
+
+### Bonus
+- Add `validator.md` if honest validator behavior changes with the new feature.
+
+### Need help?
+You can tag spec elves for cleaning up your PR. 🧚