Merge pull request #1645 from ethereum/dev

Release v0.11.0 into master
ethereum · Mar 11, 2020 · e2bd8c7 · e2bd8c7
2 parents 3daca54 + d4ce5f2
commit e2bd8c7
Show file tree

Hide file tree

Showing 111 changed files with 3,254 additions and 4,352 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -35,32 +35,45 @@ commands:
     description: "Restore the cache with pyspec keys"
     steps:
       - restore_cached_venv:
-          venv_name: v7-pyspec
-          reqs_checksum: cache-{{ checksum "tests/core/pyspec/requirements.txt" }}-{{ checksum "tests/core/pyspec/requirements-testing.txt" }}
+          venv_name: v19-pyspec
+          reqs_checksum: cache-{{ checksum "setup.py" }}
   save_pyspec_cached_venv:
     description: Save a venv into a cache with pyspec keys"
     steps:
       - save_cached_venv:
-          venv_name: v7-pyspec
-          reqs_checksum: cache-{{ checksum "tests/core/pyspec/requirements.txt" }}-{{ checksum "tests/core/pyspec/requirements-testing.txt" }}
-          venv_path: ./tests/core/pyspec/venv
-  restore_deposit_contract_cached_venv:
-    description: "Restore the cache with deposit_contract keys"
+          venv_name: v19-pyspec
+          reqs_checksum: cache-{{ checksum "setup.py" }}
+          venv_path: ./venv
+  restore_deposit_contract_compiler_cached_venv:
+    description: "Restore the venv from cache for the deposit contract compiler"
     steps:
       - restore_cached_venv:
-          venv_name: v9-deposit-contract
-          reqs_checksum: cache-{{ checksum "tests/core/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }}
-  save_deposit_contract_cached_venv:
-    description: Save a venv into a cache with deposit_contract keys"
+          venv_name: v18-deposit-contract-compiler
+          reqs_checksum: cache-{{ checksum "deposit_contract/compiler/requirements.txt" }}
+  save_deposit_contract_compiler_cached_venv:
+    description: "Save the venv to cache for later use of the deposit contract compiler"
     steps:
       - save_cached_venv:
-          venv_name: v9-deposit-contract
-          reqs_checksum: cache-{{ checksum "tests/core/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }}
-          venv_path: ./deposit_contract/venv
+          venv_name: v18-deposit-contract-compiler
+          reqs_checksum: cache-{{ checksum "deposit_contract/compiler/requirements.txt" }}
+          venv_path: ./deposit_contract/compiler/venv
+  restore_deposit_contract_tester_cached_venv:
+    description: "Restore the venv from cache for the deposit contract tester"
+    steps:
+      - restore_cached_venv:
+          venv_name: v19-deposit-contract-tester
+          reqs_checksum: cache-{{ checksum "setup.py" }}-{{ checksum "deposit_contract/tester/requirements.txt" }}
+  save_deposit_contract_tester_cached_venv:
+    description: "Save the venv to cache for later use of the deposit contract tester"
+    steps:
+      - save_cached_venv:
+          venv_name: v19-deposit-contract-tester
+          reqs_checksum: cache-{{ checksum "setup.py" }}-{{ checksum "deposit_contract/tester/requirements.txt" }}
+          venv_path: ./deposit_contract/tester/venv
 jobs:
   checkout_specs:
     docker:
-      - image: circleci/python:3.6
+      - image: circleci/python:3.8
     working_directory: ~/specs-repo
     steps:
       # Restore git repo at point close to target branch/revision, to speed up checkout
@@ -80,7 +93,7 @@ jobs:
             - ~/specs-repo
   install_pyspec_test:
     docker:
-      - image: circleci/python:3.6
+      - image: circleci/python:3.8
     working_directory: ~/specs-repo
     steps:
       - restore_cache:
@@ -92,7 +105,7 @@ jobs:
       - save_pyspec_cached_venv
   test:
     docker:
-      - image: circleci/python:3.6
+      - image: circleci/python:3.8
     working_directory: ~/specs-repo
     steps:
       - restore_cache:
@@ -114,7 +127,7 @@ jobs:
           command: sudo npm install -g doctoc && make check_toc
   codespell:
     docker:
-      - image: circleci/python:3.6
+      - image: circleci/python:3.8
     working_directory: ~/specs-repo
     steps:
       - checkout
@@ -123,7 +136,7 @@ jobs:
           command: pip install codespell --user &&  make codespell
   lint:
     docker:
-      - image: circleci/python:3.6
+      - image: circleci/python:3.8
     working_directory: ~/specs-repo
     steps:
       - restore_cache:
@@ -132,29 +145,54 @@ jobs:
       - run:
           name: Run linter
           command: make lint
-  install_deposit_contract_test:
+  install_deposit_contract_compiler:
+    docker:
+        # The deposit contract compiler is pinned to python 3.7 because of the vyper version pin.
+        - image: circleci/python:3.7
+    working_directory: ~/specs-repo
+    steps:
+      - restore_cache:
+          key: v2-specs-repo-{{ .Branch }}-{{ .Revision }}
+      - restore_deposit_contract_compiler_cached_venv
+      - run:
+          name: Install deposit contract compiler requirements
+          command: make install_deposit_contract_compiler
+      - save_deposit_contract_compiler_cached_venv
+  install_deposit_contract_tester:
     docker:
-      - image: circleci/python:3.6
+      - image: circleci/python:3.8
     working_directory: ~/specs-repo
     steps:
       - restore_cache:
           key: v2-specs-repo-{{ .Branch }}-{{ .Revision }}
-      - restore_deposit_contract_cached_venv
+      - restore_deposit_contract_tester_cached_venv
       - run:
-          name: Install deposit contract requirements
-          command: make install_deposit_contract_test
-      - save_deposit_contract_cached_venv
-  deposit_contract:
+          name: Install deposit contract tester requirements
+          command: make install_deposit_contract_tester
+      - save_deposit_contract_tester_cached_venv
+  test_compile_deposit_contract:
     docker:
-      - image: circleci/python:3.6
+      - image: circleci/python:3.7
     working_directory: ~/specs-repo
     steps:
       - restore_cache:
           key: v2-specs-repo-{{ .Branch }}-{{ .Revision }}
-      - restore_deposit_contract_cached_venv
+      - restore_deposit_contract_compiler_cached_venv
+      - run:
+          name: Run deposit contract compile test
+          command: make test_compile_deposit_contract
+  test_deposit_contract:
+    docker:
+      - image: circleci/python:3.8
+    working_directory: ~/specs-repo
+    steps:
+      - restore_cache:
+          key: v2-specs-repo-{{ .Branch }}-{{ .Revision }}
+      - restore_deposit_contract_tester_cached_venv
       - run:
           name: Run deposit contract test
           command: make test_deposit_contract
+
 workflows:
   version: 2.1
   test_spec:
@@ -171,9 +209,15 @@ workflows:
       - lint:
           requires:
             - test
-      - install_deposit_contract_test:
+      - install_deposit_contract_compiler:
+          requires:
+            - checkout_specs
+      - test_compile_deposit_contract:
+          requires:
+            - install_deposit_contract_compiler
+      - install_deposit_contract_tester:
           requires:
             - checkout_specs
-      - deposit_contract:
+      - test_deposit_contract:
           requires:
-            - install_deposit_contract_test
+            - install_deposit_contract_tester
diff --git a/.gitignore b/.gitignore
@@ -7,19 +7,24 @@ venv
 
 build/
 output/
+dist/
 
 eth2.0-spec-tests/
 
 .pytest_cache
 .mypy_cache
 
 # Dynamically built from Markdown spec
-tests/core/pyspec/eth2spec/phase0/spec.py
-tests/core/pyspec/eth2spec/phase1/spec.py
+tests/core/pyspec/eth2spec/phase0/
+tests/core/pyspec/eth2spec/phase1/
 
 # coverage reports
 .htmlcov
 .coverage
+.coverage.*
 
 # local CI testing output
 tests/core/pyspec/test-reports
+tests/core/pyspec/eth2spec/test_results.xml
+
+*.egg-info
diff --git a/Makefile b/Makefile
@@ -1,11 +1,11 @@
 SPEC_DIR = ./specs
 SSZ_DIR = ./ssz
-SCRIPT_DIR = ./scripts
 TEST_LIBS_DIR = ./tests/core
 PY_SPEC_DIR = $(TEST_LIBS_DIR)/pyspec
-TEST_VECTOR_DIR = ./eth2.0-spec-tests/tests
+TEST_VECTOR_DIR = ../eth2.0-spec-tests/tests
 GENERATOR_DIR = ./tests/generators
-DEPOSIT_CONTRACT_DIR = ./deposit_contract
+DEPOSIT_CONTRACT_COMPILER_DIR = ./deposit_contract/compiler
+DEPOSIT_CONTRACT_TESTER_DIR = ./deposit_contract/tester
 CONFIGS_DIR = ./configs
 
 # Collect a list of generator names
@@ -17,57 +17,69 @@ GENERATOR_VENVS = $(patsubst $(GENERATOR_DIR)/%, $(GENERATOR_DIR)/%venv, $(GENER
 # To check generator matching:
 #$(info $$GENERATOR_TARGETS is [${GENERATOR_TARGETS}])
 
-PHASE0_SPEC_DIR = $(SPEC_DIR)/phase0
-PY_SPEC_PHASE_0_TARGETS = $(PY_SPEC_DIR)/eth2spec/phase0/spec.py
-PY_SPEC_PHASE_0_DEPS = $(wildcard $(SPEC_DIR)/phase0/*.md)
-
-PHASE1_SPEC_DIR = $(SPEC_DIR)/phase1
-PY_SPEC_PHASE_1_TARGETS = $(PY_SPEC_DIR)/eth2spec/phase1/spec.py
-PY_SPEC_PHASE_1_DEPS = $(wildcard $(SPEC_DIR)/phase1/*.md)
-
-PY_SPEC_ALL_DEPS = $(PY_SPEC_PHASE_0_DEPS) $(PY_SPEC_PHASE_1_DEPS)
-
-PY_SPEC_ALL_TARGETS = $(PY_SPEC_PHASE_0_TARGETS) $(PY_SPEC_PHASE_1_TARGETS)
-
-MARKDOWN_FILES = $(PY_SPEC_ALL_DEPS) $(wildcard $(SPEC_DIR)/*.md) $(wildcard $(SSZ_DIR)/*.md) $(wildcard $(SPEC_DIR)/networking/*.md) $(wildcard $(SPEC_DIR)/validator/*.md)
+MARKDOWN_FILES = $(wildcard $(SPEC_DIR)/phase0/*.md) $(wildcard $(SPEC_DIR)/phase1/*.md) $(wildcard $(SSZ_DIR)/*.md) $(wildcard $(SPEC_DIR)/networking/*.md) $(wildcard $(SPEC_DIR)/validator/*.md)
 
 COV_HTML_OUT=.htmlcov
 COV_INDEX_FILE=$(PY_SPEC_DIR)/$(COV_HTML_OUT)/index.html
 
-.PHONY: clean partial_clean all test citest lint generate_tests pyspec phase0 phase1 install_test open_cov \
-        install_deposit_contract_test test_deposit_contract compile_deposit_contract check_toc
+.PHONY: clean partial_clean all test citest lint generate_tests pyspec install_test open_cov \
+        install_deposit_contract_tester test_deposit_contract install_deposit_contract_compiler \
+        compile_deposit_contract test_compile_deposit_contract check_toc
 
 all: $(PY_SPEC_ALL_TARGETS)
 
 # deletes everything except the venvs
 partial_clean:
 	rm -rf $(TEST_VECTOR_DIR)
 	rm -rf $(GENERATOR_VENVS)
+	rm -rf .pytest_cache
+	rm -f .coverage
 	rm -rf $(PY_SPEC_DIR)/.pytest_cache
-	rm -rf $(PY_SPEC_ALL_TARGETS)
-	rm -rf $(DEPOSIT_CONTRACT_DIR)/.pytest_cache
+	rm -rf $(DEPOSIT_CONTRACT_COMPILER_DIR)/.pytest_cache
+	rm -rf $(DEPOSIT_CONTRACT_TESTER_DIR)/.pytest_cache
+	rm -rf $(PY_SPEC_DIR)/phase0
+	rm -rf $(PY_SPEC_DIR)/phase1
 	rm -rf $(PY_SPEC_DIR)/$(COV_HTML_OUT)
 	rm -rf $(PY_SPEC_DIR)/.coverage
 	rm -rf $(PY_SPEC_DIR)/test-reports
+	rm -rf eth2spec.egg-info dist build
+
 
 clean: partial_clean
+	rm -rf venv
 	rm -rf $(PY_SPEC_DIR)/venv
-	rm -rf $(DEPOSIT_CONTRACT_DIR)/venv
+	rm -rf $(DEPOSIT_CONTRACT_COMPILER_DIR)/venv
+	rm -rf $(DEPOSIT_CONTRACT_TESTER_DIR)/venv
+
+# The pyspec is rebuilt to enforce the /specs being part of eth2specs source distribution. It could be forgotten otherwise.
+dist_build: pyspec
+	python3 setup.py sdist bdist_wheel
+
+dist_check:
+	python3 -m twine check dist/*
+
+dist_upload:
+	python3 -m twine upload dist/*
+
 
 # "make generate_tests" to run all generators
-generate_tests: $(PY_SPEC_ALL_TARGETS) $(GENERATOR_TARGETS)
+generate_tests: $(GENERATOR_TARGETS)
+
+# "make pyspec" to create the pyspec for all phases.
+pyspec:
+	. venv/bin/activate; python3 setup.py pyspecdev
 
 # installs the packages to run pyspec tests
 install_test:
-	cd $(PY_SPEC_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install -r requirements-testing.txt;
+	python3 -m venv venv; . venv/bin/activate; pip3 install .[test] .[lint]
 
-test: $(PY_SPEC_ALL_TARGETS)
-	cd $(PY_SPEC_DIR); . venv/bin/activate;	export PYTHONPATH="./"; \
+test: pyspec
+	. venv/bin/activate; cd $(PY_SPEC_DIR); \
 	python -m pytest -n 4 --cov=eth2spec.phase0.spec --cov=eth2spec.phase1.spec --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec
 
-citest: $(PY_SPEC_ALL_TARGETS)
-	cd $(PY_SPEC_DIR); mkdir -p test-reports/eth2spec; . venv/bin/activate; \
-	python -m pytest -n 4 --junitxml=test-reports/eth2spec/test_results.xml eth2spec
+citest: pyspec
+	mkdir -p tests/core/pyspec/test-reports/eth2spec; . venv/bin/activate; cd $(PY_SPEC_DIR); \
+	python -m pytest -n 4 --junitxml=eth2spec/test_results.xml eth2spec
 
 open_cov:
 	((open "$(COV_INDEX_FILE)" || xdg-open "$(COV_INDEX_FILE)") &> /dev/null) &
@@ -83,31 +95,29 @@ check_toc: $(MARKDOWN_FILES:=.toc)
 codespell:
 	codespell . --skip ./.git -I .codespell-whitelist
 
-lint: $(PY_SPEC_ALL_TARGETS)
-	cd $(PY_SPEC_DIR); . venv/bin/activate; \
+lint: pyspec
+	. venv/bin/activate; cd $(PY_SPEC_DIR); \
 	flake8  --ignore=E252,W504,W503 --max-line-length=120 ./eth2spec \
 	&& cd ./eth2spec && mypy --follow-imports=silent --warn-unused-ignores --ignore-missing-imports --check-untyped-defs --disallow-incomplete-defs --disallow-untyped-defs -p phase0 \
 	&& mypy --follow-imports=silent --warn-unused-ignores --ignore-missing-imports --check-untyped-defs --disallow-incomplete-defs --disallow-untyped-defs -p phase1;
 
-install_deposit_contract_test: $(PY_SPEC_ALL_TARGETS)
-	cd $(DEPOSIT_CONTRACT_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install -r requirements-testing.txt
-
-compile_deposit_contract:
-	cd $(DEPOSIT_CONTRACT_DIR); . venv/bin/activate; \
-	python tool/compile_deposit_contract.py contracts/validator_registration.vy;
+install_deposit_contract_tester:
+	cd $(DEPOSIT_CONTRACT_TESTER_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install -r requirements.txt
 
 test_deposit_contract:
-	cd $(DEPOSIT_CONTRACT_DIR); . venv/bin/activate; \
+	cd $(DEPOSIT_CONTRACT_TESTER_DIR); . venv/bin/activate; \
 	python -m pytest .
 
-# "make pyspec" to create the pyspec for all phases.
-pyspec: $(PY_SPEC_ALL_TARGETS)
+install_deposit_contract_compiler:
+	cd $(DEPOSIT_CONTRACT_COMPILER_DIR); python3.7 -m venv venv; . venv/bin/activate; pip3.7 install -r requirements.txt
 
-$(PY_SPEC_PHASE_0_TARGETS): $(PY_SPEC_PHASE_0_DEPS)
-	python3 $(SCRIPT_DIR)/build_spec.py -p0 $(PHASE0_SPEC_DIR)/beacon-chain.md $(PHASE0_SPEC_DIR)/fork-choice.md $(PHASE0_SPEC_DIR)/validator.md $@
+compile_deposit_contract:
+	cd $(DEPOSIT_CONTRACT_COMPILER_DIR); . venv/bin/activate; \
+	python3.7 deposit_contract/compile.py contracts/validator_registration.vy
 
-$(PY_SPEC_DIR)/eth2spec/phase1/spec.py: $(PY_SPEC_PHASE_1_DEPS)
-	python3 $(SCRIPT_DIR)/build_spec.py -p1 $(PHASE0_SPEC_DIR)/beacon-chain.md $(PHASE0_SPEC_DIR)/fork-choice.md $(SSZ_DIR)/merkle-proofs.md $(PHASE1_SPEC_DIR)/custody-game.md $(PHASE1_SPEC_DIR)/shard-data-chains.md $(PHASE1_SPEC_DIR)/beacon-chain-misc.md $@
+test_compile_deposit_contract:
+	cd $(DEPOSIT_CONTRACT_COMPILER_DIR); . venv/bin/activate; \
+	python3.7 -m pytest .
 
 CURRENT_DIR = ${CURDIR}
 
@@ -141,5 +151,5 @@ $(TEST_VECTOR_DIR)/:
 
 # For any generator, build it using the run_generator function.
 # (creation of output dir is a dependency)
-gen_%: $(PY_SPEC_ALL_TARGETS) $(TEST_VECTOR_DIR)
+gen_%: $(TEST_VECTOR_DIR)
 	$(call run_generator,$*)
diff --git a/README.md b/README.md
@@ -16,11 +16,13 @@ Core specifications for Eth2 clients be found in [specs/](specs/). These are div
 * [Fork Choice](specs/phase0/fork-choice.md)
 * [Deposit Contract](specs/phase0/deposit-contract.md)
 * [Honest Validator](specs/phase0/validator.md)
+* [P2P Networking](specs/phase0/p2p-interface.md)
 
 ### Phase 1
+* [From Phase 0 to Phase 1](specs/phase1/phase1-fork.md)
+* [The Beacon Chain for Shards](specs/phase1/beacon-chain.md)
 * [Custody Game](specs/phase1/custody-game.md)
-* [Shard Data Chains](specs/phase1/shard-data-chains.md)
-* [Misc beacon chain updates](specs/phase1/beacon-chain-misc.md)
+* [Shard Transition and Fraud Proofs](specs/phase1/fraud-proofs.md)
 * [Light client syncing protocol](specs/phase1/light-client-sync.md)
 
 ### Phase 2
@@ -57,11 +59,11 @@ The following are the broad design goals for Ethereum 2.0:
 
 * [Design Rationale](https://notes.ethereum.org/s/rkhCgQteN#)
 * [Phase 0 Onboarding Document](https://notes.ethereum.org/s/Bkn3zpwxB)
+* [Combining GHOST and Casper paper](https://arxiv.org/abs/2003.03052)
 
 
 ## For spec contributors
 
 Documentation on the different components used during spec writing can be found here:
 * [YAML Test Generators](tests/generators/README.md)
 * [Executable Python Spec, with Py-tests](tests/core/pyspec/README.md)
-