reorg specs by fork and move ssz out to own folder. make all of the b…

…uild and link changes to support move

.circleci/config.yml

@@ -35,27 +35,27 @@ commands:
     description: "Restore the cache with pyspec keys"
     steps:
       - restore_cached_venv:
-          venv_name: v5-pyspec
-          reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}
+          venv_name: v6-pyspec
+          reqs_checksum: cache-{{ checksum "tests/core/pyspec/requirements.txt" }}-{{ checksum "tests/core/pyspec/requirements-testing.txt" }}
   save_pyspec_cached_venv:
     description: Save a venv into a cache with pyspec keys"
     steps:
       - save_cached_venv:
-          venv_name: v5-pyspec
-          reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}
-          venv_path: ./test_libs/pyspec/venv
+          venv_name: v6-pyspec
+          reqs_checksum: cache-{{ checksum "tests/core/pyspec/requirements.txt" }}-{{ checksum "tests/core/pyspec/requirements-testing.txt" }}
+          venv_path: ./tests/core/pyspec/venv
   restore_deposit_contract_cached_venv:
     description: "Restore the cache with deposit_contract keys"
     steps:
       - restore_cached_venv:
-          venv_name: v8-deposit-contract
-          reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }}
+          venv_name: v9-deposit-contract
+          reqs_checksum: cache-{{ checksum "tests/core/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }}
   save_deposit_contract_cached_venv:
     description: Save a venv into a cache with deposit_contract keys"
     steps:
       - save_cached_venv:
-          venv_name: v8-deposit-contract
-          reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }}
+          venv_name: v9-deposit-contract
+          reqs_checksum: cache-{{ checksum "tests/core/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }}
           venv_path: ./deposit_contract/venv
 jobs:
   checkout_specs:
@@ -66,16 +66,16 @@ jobs:
       # Restore git repo at point close to target branch/revision, to speed up checkout
       - restore_cache:
           keys:
-            - v1-specs-repo-{{ .Branch }}-{{ .Revision }}
-            - v1-specs-repo-{{ .Branch }}-
-            - v1-specs-repo-
+            - v2-specs-repo-{{ .Branch }}-{{ .Revision }}
+            - v2-specs-repo-{{ .Branch }}-
+            - v2-specs-repo-
       - checkout
       - run:
           name: Clean up git repo to reduce cache size
           command: git gc
       # Save the git checkout as a cache, to make cloning next time faster.
       - save_cache:
-          key: v1-specs-repo-{{ .Branch }}-{{ .Revision }}
+          key: v2-specs-repo-{{ .Branch }}-{{ .Revision }}
           paths:
             - ~/specs-repo
   install_pyspec_test:
@@ -84,7 +84,7 @@ jobs:
     working_directory: ~/specs-repo
     steps:
       - restore_cache:
-          key: v1-specs-repo-{{ .Branch }}-{{ .Revision }}
+          key: v2-specs-repo-{{ .Branch }}-{{ .Revision }}
       - restore_pyspec_cached_venv
       - run:
           name: Install pyspec requirements
@@ -96,13 +96,13 @@ jobs:
     working_directory: ~/specs-repo
     steps:
       - restore_cache:
-          key: v1-specs-repo-{{ .Branch }}-{{ .Revision }}
+          key: v2-specs-repo-{{ .Branch }}-{{ .Revision }}
       - restore_pyspec_cached_venv
       - run:
           name: Run py-tests
           command: make citest
       - store_test_results:
-          path: test_libs/pyspec/test-reports
+          path: tests/core/pyspec/test-reports
   table_of_contents:
     docker:
       - image: circleci/node:10.16.3
@@ -127,7 +127,7 @@ jobs:
     working_directory: ~/specs-repo
     steps:
       - restore_cache:
-          key: v1-specs-repo-{{ .Branch }}-{{ .Revision }}
+          key: v2-specs-repo-{{ .Branch }}-{{ .Revision }}
       - restore_pyspec_cached_venv
       - run:
           name: Run linter
@@ -138,7 +138,7 @@ jobs:
     working_directory: ~/specs-repo
     steps:
       - restore_cache:
-          key: v1-specs-repo-{{ .Branch }}-{{ .Revision }}
+          key: v2-specs-repo-{{ .Branch }}-{{ .Revision }}
       - restore_deposit_contract_cached_venv
       - run:
           name: Install deposit contract requirements
@@ -150,7 +150,7 @@ jobs:
     working_directory: ~/specs-repo
     steps:
       - restore_cache:
-          key: v1-specs-repo-{{ .Branch }}-{{ .Revision }}
+          key: v2-specs-repo-{{ .Branch }}-{{ .Revision }}
       - restore_deposit_contract_cached_venv
       - run:
           name: Run deposit contract test

.gitignore

@@ -14,12 +14,12 @@ eth2.0-spec-tests/
 .mypy_cache
 
 # Dynamically built from Markdown spec
-test_libs/pyspec/eth2spec/phase0/spec.py
-test_libs/pyspec/eth2spec/phase1/spec.py
+tests/core/pyspec/eth2spec/phase0/spec.py
+tests/core/pyspec/eth2spec/phase1/spec.py
 
 # coverage reports
 .htmlcov
 .coverage
 
 # local CI testing output
-test_libs/pyspec/test-reports
+tests/core/pyspec/test-reports

Makefile

@@ -1,9 +1,10 @@
 SPEC_DIR = ./specs
+SSZ_DIR = ./ssz
 SCRIPT_DIR = ./scripts
-TEST_LIBS_DIR = ./test_libs
+TEST_LIBS_DIR = ./tests/core
 PY_SPEC_DIR = $(TEST_LIBS_DIR)/pyspec
 TEST_VECTOR_DIR = ./eth2.0-spec-tests/tests
-GENERATOR_DIR = ./test_generators
+GENERATOR_DIR = ./tests/generators
 DEPOSIT_CONTRACT_DIR = ./deposit_contract
 CONFIGS_DIR = ./configs
 
@@ -16,17 +17,19 @@ GENERATOR_VENVS = $(patsubst $(GENERATOR_DIR)/%, $(GENERATOR_DIR)/%venv, $(GENER
 # To check generator matching:
 #$(info $$GENERATOR_TARGETS is [${GENERATOR_TARGETS}])
 
+PHASE0_SPEC_DIR = $(SPEC_DIR)/phase0
 PY_SPEC_PHASE_0_TARGETS = $(PY_SPEC_DIR)/eth2spec/phase0/spec.py
-PY_SPEC_PHASE_0_DEPS = $(wildcard $(SPEC_DIR)/core/0_*.md)
+PY_SPEC_PHASE_0_DEPS = $(wildcard $(SPEC_DIR)/phase0/*.md)
 
+PHASE1_SPEC_DIR = $(SPEC_DIR)/phase1
 PY_SPEC_PHASE_1_TARGETS = $(PY_SPEC_DIR)/eth2spec/phase1/spec.py
-PY_SPEC_PHASE_1_DEPS = $(wildcard $(SPEC_DIR)/core/1_*.md)
+PY_SPEC_PHASE_1_DEPS = $(wildcard $(SPEC_DIR)/phase1/*.md)
 
 PY_SPEC_ALL_DEPS = $(PY_SPEC_PHASE_0_DEPS) $(PY_SPEC_PHASE_1_DEPS)
 
 PY_SPEC_ALL_TARGETS = $(PY_SPEC_PHASE_0_TARGETS) $(PY_SPEC_PHASE_1_TARGETS)
 
-MARKDOWN_FILES = $(PY_SPEC_ALL_DEPS) $(wildcard $(SPEC_DIR)/*.md) $(wildcard $(SPEC_DIR)/light_client/*.md) $(wildcard $(SPEC_DIR)/networking/*.md) $(wildcard $(SPEC_DIR)/validator/*.md)
+MARKDOWN_FILES = $(PY_SPEC_ALL_DEPS) $(wildcard $(SPEC_DIR)/*.md) $(wildcard $(SSZ_DIR)/*.md) $(wildcard $(SPEC_DIR)/networking/*.md) $(wildcard $(SPEC_DIR)/validator/*.md)
 
 COV_HTML_OUT=.htmlcov
 COV_INDEX_FILE=$(PY_SPEC_DIR)/$(COV_HTML_OUT)/index.html
@@ -101,10 +104,10 @@ test_deposit_contract:
 pyspec: $(PY_SPEC_ALL_TARGETS)
 
 $(PY_SPEC_PHASE_0_TARGETS): $(PY_SPEC_PHASE_0_DEPS)
-	python3 $(SCRIPT_DIR)/build_spec.py -p0 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/0_fork-choice.md $(SPEC_DIR)/validator/0_beacon-chain-validator.md $@
+	python3 $(SCRIPT_DIR)/build_spec.py -p0 $(PHASE0_SPEC_DIR)/beacon-chain.md $(PHASE0_SPEC_DIR)/fork-choice.md $(PHASE0_SPEC_DIR)/validator.md $@
 
 $(PY_SPEC_DIR)/eth2spec/phase1/spec.py: $(PY_SPEC_PHASE_1_DEPS)
-	python3 $(SCRIPT_DIR)/build_spec.py -p1 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/0_fork-choice.md $(SPEC_DIR)/light_client/merkle_proofs.md $(SPEC_DIR)/core/1_custody-game.md $(SPEC_DIR)/core/1_shard-data-chains.md $(SPEC_DIR)/core/1_beacon-chain-misc.md $@
+	python3 $(SCRIPT_DIR)/build_spec.py -p1 $(PHASE0_SPEC_DIR)/beacon-chain.md $(PHASE0_SPEC_DIR)/fork-choice.md $(SSZ_DIR)/merkle-proofs.md $(PHASE1_SPEC_DIR)/custody-game.md $(PHASE1_SPEC_DIR)/shard-data-chains.md $(PHASE1_SPEC_DIR)/beacon-chain-misc.md $@
 
 CURRENT_DIR = ${CURDIR}
 

README.md

@@ -9,18 +9,19 @@ This repository hosts the current Eth2 specifications. Discussions about design
 
 ## Specs
 
-Core specifications for Eth2 client validation can be found in [specs/core](specs/core). These are divided into phases. Each subsequent phase depends upon the prior. The current phases specified are:
+Core specifications for Eth2 clients be found in [specs/](specs/). These are divided into phases. Each subsequent phase depends upon the prior. The current phases specified are:
 
 ### Phase 0
-* [The Beacon Chain](specs/core/0_beacon-chain.md)
-* [Fork Choice](specs/core/0_fork-choice.md)
-* [Deposit Contract](specs/core/0_deposit-contract.md)
-* [Honest Validator](specs/validator/0_beacon-chain-validator.md)
+* [The Beacon Chain](specs/phase0/beacon-chain.md)
+* [Fork Choice](specs/phase0/fork-choice.md)
+* [Deposit Contract](specs/phase0/deposit-contract.md)
+* [Honest Validator](specs/phase0/validator.md)
 
 ### Phase 1
-* [Custody Game](specs/core/1_custody-game.md)
-* [Shard Data Chains](specs/core/1_shard-data-chains.md)
-* [Misc beacon chain updates](specs/core/1_beacon-chain-misc.md)
+* [Custody Game](specs/phase1/custody-game.md)
+* [Shard Data Chains](specs/phase1/shard-data-chains.md)
+* [Misc beacon chain updates](specs/phase1/beacon-chain-misc.md)
+* [Light client syncing protocol](specs/phase1/light-client-sync.md)
 
 ### Phase 2
 
@@ -30,11 +31,9 @@ See the [Eth2 Phase 2 Wiki](https://hackmd.io/UzysWse1Th240HELswKqVA?view) for c
 
 ### Accompanying documents can be found in [specs](specs) and include:
 
-* [SimpleSerialize (SSZ) spec](specs/simple-serialize.md)
-* [BLS signature verification](specs/bls_signature.md)
-* [General test format](specs/test_formats/README.md)
-* [Merkle proof formats](specs/light_client/merkle_proofs.md)
-* [Light client syncing protocol](specs/light_client/sync_protocol.md)
+* [SimpleSerialize (SSZ) spec](ssz/simple-serialize.md)
+* [Merkle proof formats](ssz/merkle-proofs.md)
+* [General test format](tests/formats/README.md)
 
 ## Additional specifications for client implementers
 
@@ -63,6 +62,6 @@ The following are the broad design goals for Ethereum 2.0:
 ## For spec contributors
 
 Documentation on the different components used during spec writing can be found here:
-* [YAML Test Generators](test_generators/README.md)
-* [Executable Python Spec, with Py-tests](test_libs/pyspec/README.md)
+* [YAML Test Generators](tests/generators/README.md)
+* [Executable Python Spec, with Py-tests](tests/core/pyspec/README.md)
 

deposit_contract/requirements-testing.txt

@@ -2,4 +2,4 @@ eth-tester[py-evm]==0.1.0b39
 git+https://github.com/vyperlang/vyper@1761-HOTFIX-v0.1.0-beta.13
 web3==5.0.0b2
 pytest==3.6.1
-../test_libs/pyspec
+../tests/core/pyspec

scripts/build_spec.py

@@ -308,18 +308,18 @@ def build_phase1_spec(phase0_beacon_sourcefile: str,
     description = '''
 Build the specs from the md docs.
 If building phase 0:
-    1st argument is input /core/0_beacon-chain.md
-    2nd argument is input /core/0_fork-choice.md
-    3rd argument is input /core/0_beacon-chain-validator.md
+    1st argument is input phase0/beacon-chain.md
+    2nd argument is input phase0/fork-choice.md
+    3rd argument is input phase0/validator.md
     4th argument is output spec.py
 
 If building phase 1:
-    1st argument is input /core/0_beacon-chain.md
-    2nd argument is input /core/0_fork-choice.md
-    3rd argument is input /light_client/merkle_proofs.md
-    4th argument is input /core/1_custody-game.md
-    5th argument is input /core/1_shard-data-chains.md
-    6th argument is input /core/1_beacon-chain-misc.md
+    1st argument is input phase0/beacon-chain.md
+    2nd argument is input phase0/fork-choice.md
+    3rd argument is input ssz/merkle-proofs.md
+    4th argument is input phase1/custody-game.md
+    5th argument is input phase1/shard-data-chains.md
+    6th argument is input phase1/beacon-chain-misc.md
     7th argument is output spec.py
 '''
     parser = ArgumentParser(description=description)
@@ -338,9 +338,9 @@ def build_phase1_spec(phase0_beacon_sourcefile: str,
         else:
             print(
                 " Phase 1 requires input files as well as an output file:\n"
-                "\t core/phase_0: (0_beacon-chain.md, 0_fork-choice.md)\n"
-                "\t light_client: (merkle_proofs.md)\n"
-                "\t core/phase_1: (1_custody-game.md, 1_shard-data-chains.md, 1_beacon-chain-misc.md)\n"
+                "\t phase0: (beacon-chain.md, fork-choice.md)\n"
+                "\t ssz: (merkle-proofs.md)\n"
+                "\t phase1: (custody-game.md, shard-data-chains.md, beacon-chain-misc.md)\n"
                 "\t and output.py"
             )
     else:

specs/core/0_beacon-chain.md → specs/phase0/beacon-chain.md

@@ -261,7 +261,7 @@ The following values are (non-configurable) constants used throughout the specif
 
 ## Containers
 
-The following types are [SimpleSerialize (SSZ)](../simple-serialize.md) containers.
+The following types are [SimpleSerialize (SSZ)](../../ssz/simple-serialize.md) containers.
 
 *Note*: The definitions are ordered topologically to facilitate execution of the spec.
 
@@ -579,7 +579,7 @@ def bytes_to_int(data: bytes) -> uint64:
 
 #### `hash_tree_root`
 
-`def hash_tree_root(object: SSZSerializable) -> Root` is a function for hashing objects into a single root by utilizing a hash tree structure, as defined in the [SSZ spec](../simple-serialize.md#merkleization).
+`def hash_tree_root(object: SSZSerializable) -> Root` is a function for hashing objects into a single root by utilizing a hash tree structure, as defined in the [SSZ spec](../../ssz/simple-serialize.md#merkleization).
 
 #### BLS Signatures
 

specs/core/0_deposit-contract.md → specs/phase0/deposit-contract.md

@@ -40,7 +40,7 @@ The initial deployment phases of Ethereum 2.0 are implemented without consensus
 
 ### `deposit` function
 
-The deposit contract has a public `deposit` function to make deposits. It takes as arguments `pubkey: bytes[48], withdrawal_credentials: bytes[32], signature: bytes[96], deposit_data_root: bytes32`. The first three arguments populate a [`DepositData`](./0_beacon-chain.md#depositdata) object, and `deposit_data_root` is the expected `DepositData` root as a protection against malformatted calldata.
+The deposit contract has a public `deposit` function to make deposits. It takes as arguments `pubkey: bytes[48], withdrawal_credentials: bytes[32], signature: bytes[96], deposit_data_root: bytes32`. The first three arguments populate a [`DepositData`](./beacon-chain.md#depositdata) object, and `deposit_data_root` is the expected `DepositData` root as a protection against malformatted calldata.
 
 #### Deposit amount
 

specs/core/0_fork-choice.md → specs/phase0/fork-choice.md

@@ -34,7 +34,7 @@
 
 ## Introduction
 
-This document is the beacon chain fork choice spec, part of Ethereum 2.0 Phase 0. It assumes the [beacon chain state transition function spec](./0_beacon-chain.md).
+This document is the beacon chain fork choice spec, part of Ethereum 2.0 Phase 0. It assumes the [beacon chain state transition function spec](./beacon-chain.md).
 
 ## Fork choice
 
@@ -48,7 +48,7 @@ The head block root associated with a `store` is defined as `get_head(store)`. A
 
 1) **Leap seconds**: Slots will last `SECONDS_PER_SLOT + 1` or `SECONDS_PER_SLOT - 1` seconds around leap seconds. This is automatically handled by [UNIX time](https://en.wikipedia.org/wiki/Unix_time).
 2) **Honest clocks**: Honest nodes are assumed to have clocks synchronized within `SECONDS_PER_SLOT` seconds of each other.
-3) **Eth1 data**: The large `ETH1_FOLLOW_DISTANCE` specified in the [honest validator document](../validator/0_beacon-chain-validator.md) should ensure that `state.latest_eth1_data` of the canonical Ethereum 2.0 chain remains consistent with the canonical Ethereum 1.0 chain. If not, emergency manual intervention will be required.
+3) **Eth1 data**: The large `ETH1_FOLLOW_DISTANCE` specified in the [honest validator document](./validator.md) should ensure that `state.latest_eth1_data` of the canonical Ethereum 2.0 chain remains consistent with the canonical Ethereum 1.0 chain. If not, emergency manual intervention will be required.
 4) **Manual forks**: Manual forks may arbitrarily change the fork choice rule but are expected to be enacted at epoch transitions, with the fork details reflected in `state.fork`.
 5) **Implementation**: The implementation found in this specification is constructed for ease of understanding rather than for optimization in computation, space, or any other resource. A number of optimized alternatives can be found [here](https://github.com/protolambda/lmd-ghost).
 

specs/networking/p2p-interface.md → specs/phase0/p2p-interface.md

@@ -398,12 +398,12 @@ Here, `result` represents the 1-byte response code.
 
 The token of the negotiated protocol ID specifies the type of encoding to be used for the req/resp interaction. Two values are possible at this time:
 
--  `ssz`: the contents are [SSZ-encoded](../simple-serialize.md). This encoding type MUST be supported by all clients. For objects containing a single field, only the field is SSZ-encoded not a container with a single field. For example, the `BeaconBlocksByRoot` request is an SSZ-encoded list of `Bytes32`'s.
+-  `ssz`: the contents are [SSZ-encoded](../../ssz/simple-serialize.md). This encoding type MUST be supported by all clients. For objects containing a single field, only the field is SSZ-encoded not a container with a single field. For example, the `BeaconBlocksByRoot` request is an SSZ-encoded list of `Bytes32`'s.
 -  `ssz_snappy`: The contents are SSZ-encoded and then compressed with [Snappy](https://github.com/google/snappy). MAY be supported in the interoperability testnet; MUST be supported in mainnet.
 
 #### SSZ-encoding strategy (with or without Snappy)
 
-The [SimpleSerialize (SSZ) specification](../simple-serialize.md) outlines how objects are SSZ-encoded. If the Snappy variant is selected, we feed the serialized form to the Snappy compressor on encoding. The inverse happens on decoding.
+The [SimpleSerialize (SSZ) specification](../../ssz/simple-serialize.md) outlines how objects are SSZ-encoded. If the Snappy variant is selected, we feed the serialized form to the Snappy compressor on encoding. The inverse happens on decoding.
 
 **Encoding-dependent header:** Req/Resp protocols using the `ssz` or `ssz_snappy` encoding strategies MUST prefix all encoded and compressed (if applicable) payloads with an unsigned [protobuf varint](https://developers.google.com/protocol-buffers/docs/encoding#varints).