Merge pull request #967 from ethereum/ci-tests-config

update CI config: caching of repo, venv, split test install/run

.circleci/config.yml

@@ -1,89 +1,97 @@
 version: 2.1
+commands:
+  restore_cached_venv:
+    description: "Restores a cached venv"
+    parameters:
+      reqs_checksum:
+        type: string
+        default: "1234"
+      venv_name:
+        type: string
+        default: "default-name"
+    steps:
+      - restore_cache:
+          keys:
+            - << parameters.venv_name >>-venv-<< parameters.reqs_checksum >>
+            # fallback to using the latest cache if no exact match is found
+            - << parameters.venv_name >>-venv-
+  save_cached_venv:
+    description: "Saves a venv into a cache"
+    parameters:
+      reqs_checksum:
+        type: string
+        default: "1234"
+      venv_path:
+        type: string
+        default: "venv"
+      venv_name:
+        type: string
+        default: "default-name"
+    steps:
+      - save_cache:
+          key: << parameters.venv_name >>-venv-<< parameters.reqs_checksum >>
+          paths: << parameters.venv_path >>
 jobs:
-  build:
+  checkout_specs:
     docker:
       - image: circleci/python:3.6
-    working_directory: ~/repo
-
+    working_directory: ~/specs-repo
     steps:
+      # Restore git repo at point close to target branch/revision, to speed up checkout
+      - restore_cache:
+          keys:
+            - v1-specs-repo-{{ .Branch }}-{{ .Revision }}
+            - v1-specs-repo-{{ .Branch }}-
+            - v1-specs-repo-
       - checkout
       - run:
-          name: Build pyspec
-          command: make pyspec
-
+          name: Clean up git repo to reduce cache size
+          command: git gc
+      # Save the git checkout as a cache, to make cloning next time faster.
+      - save_cache:
+          key: v1-specs-repo-{{ .Branch }}-{{ .Revision }}
+          paths:
+            - ~/specs-repo
+  install_test:
+    docker:
+      - image: circleci/python:3.6
+    working_directory: ~/specs-repo
+    steps:
+      - restore_cache:
+          key: v1-specs-repo-{{ .Branch }}-{{ .Revision }}
+      - restore_cached_venv:
+          venv_name: v1-pyspec
+          reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}'
+      - run:
+          name: Install pyspec requirements
+          command: make install_test
+      - save_cached_venv:
+          venv_name: v1-pyspec
+          reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}'
+          venv_path: ./test_libs/pyspec/venv
+  test:
+    docker:
+      - image: circleci/python:3.6
+    working_directory: ~/specs-repo
+    steps:
+      - restore_cache:
+          key: v1-specs-repo-{{ .Branch }}-{{ .Revision }}
+      - restore_cached_venv:
+          venv_name: v1-pyspec
+          reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}'
       - run:
           name: Run py-tests
-          command: make test
-
-# TODO see #928: decide on CI triggering of yaml tests building,
-#  and destination of output (new yaml tests LFS-configured repository)
-#
-#      - run:
-#          name: Generate YAML tests
-#          command: make gen_yaml_tests
-#
-#      - store_artifacts:
-#          path: test-reports
-#          destination: test-reports
-#
-#      - run:
-#          name: Save YAML tests for deployment
-#          command: |
-#            mkdir /tmp/workspace
-#            cp -r yaml_tests /tmp/workspace/
-#            git log -1 >> /tmp/workspace/latest_commit_message
-#      - persist_to_workspace:
-#          root: /tmp/workspace
-#          paths:
-#            - yaml_tests
-#            - latest_commit_message
-#  commit:
-#    docker:
-#      - image: circleci/python:3.6
-#    steps:
-#      - attach_workspace:
-#          at: /tmp/workspace
-#      - add_ssh_keys:
-#          fingerprints:
-#            - "01:85:b6:36:96:a6:84:72:e4:9b:4e:38:ee:21:97:fa"
-#      - run:
-#          name: Checkout test repository
-#          command: |
-#            ssh-keyscan -H github.com >> ~/.ssh/known_hosts
-#            git clone git@github.com:ethereum/eth2.0-tests.git
-#      - run:
-#          name: Commit and push generated YAML tests
-#          command: |
-#            cd eth2.0-tests
-#            git config user.name 'eth2TestGenBot'
-#            git config user.email '47188154+eth2TestGenBot@users.noreply.github.com'
-#            for filename in /tmp/workspace/yaml_tests/*; do
-#              rm -rf $(basename $filename)
-#              cp -r $filename .
-#            done
-#            git add .
-#            if git diff --cached --exit-code >& /dev/null; then
-#              echo "No changes to commit"
-#            else
-#              echo -e "Update generated tests\n\nLatest commit message from eth2.0-specs:\n" > commit_message
-#              cat /tmp/workspace/latest_commit_message >> commit_message
-#              git commit -F commit_message
-#              git push origin master
-#            fi
-#workflows:
-#  version: 2.1
-#
-#  build_and_commit:
-#    jobs:
-#      - build:
-#          filters:
-#            tags:
-#              only: /.*/
-#      - commit:
-#          requires:
-#            - build
-#          filters:
-#            tags:
-#              only: /.*/
-#            branches:
-#              ignore: /.*/
+          command: make citest
+      - store_test_results:
+          path: test_libs/pyspec/test-reports
+workflows:
+  version: 2.1
+  test_spec:
+    jobs:
+      - checkout_specs
+      - install_test:
+          requires:
+            - checkout_specs
+      - test:
+          requires:
+            - install_test

Makefile

@@ -16,7 +16,7 @@ PY_SPEC_PHASE_0_TARGETS = $(PY_SPEC_DIR)/eth2spec/phase0/spec.py
 PY_SPEC_ALL_TARGETS = $(PY_SPEC_PHASE_0_TARGETS)
 
 
-.PHONY: clean all test gen_yaml_tests pyspec phase0
+.PHONY: clean all test citest gen_yaml_tests pyspec phase0 install_test
 
 all: $(PY_SPEC_ALL_TARGETS) $(YAML_TEST_DIR) $(YAML_TEST_TARGETS)
 
@@ -27,11 +27,17 @@ clean:
 	rm -rf $(PY_SPEC_ALL_TARGETS)
 
 # "make gen_yaml_tests" to run generators
-gen_yaml_tests: $(YAML_TEST_DIR) $(YAML_TEST_TARGETS)
+gen_yaml_tests: $(PY_SPEC_ALL_TARGETS) $(YAML_TEST_DIR) $(YAML_TEST_TARGETS)
+
+# installs the packages to run pyspec tests
+install_test:
+	cd $(PY_SPEC_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install -r requirements.txt;
 
-# runs a limited set of tests against a minimal config
 test: $(PY_SPEC_ALL_TARGETS)
-	cd $(PY_SPEC_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install -r requirements.txt; python -m pytest -m minimal_config .
+	cd $(PY_SPEC_DIR); . venv/bin/activate; python -m pytest -m minimal_config .
+
+citest: $(PY_SPEC_ALL_TARGETS)
+	cd $(PY_SPEC_DIR); mkdir -p test-reports/eth2spec; . venv/bin/activate; python -m pytest --junitxml=test-reports/eth2spec/test_results.xml -m minimal_config .
 
 # "make pyspec" to create the pyspec for all phases.
 pyspec: $(PY_SPEC_ALL_TARGETS)
@@ -69,5 +75,5 @@ $(YAML_TEST_DIR):
 
 # For any target within the tests dir, build it using the build_yaml_tests function.
 # (creation of output dir is a dependency)
-$(YAML_TEST_DIR)%: $(YAML_TEST_DIR)
+$(YAML_TEST_DIR)%: $(PY_SPEC_ALL_TARGETS) $(YAML_TEST_DIR)
 	$(call build_yaml_tests,$*)

test_libs/pyspec/README.md

@@ -19,6 +19,8 @@ Or, to build a single file, specify the path, e.g. `make test_libs/pyspec/eth2sp
 
 ## Py-tests
 
+After building, you can install the dependencies for running the `pyspec` tests with `make install_test`
+
 These tests are not intended for client-consumption.
 These tests are sanity tests, to verify if the spec itself is consistent.
 
@@ -38,8 +40,9 @@ python3 -m venv venv
 . venv/bin/activate
 pip3 install -r requirements.txt
 ```
-Note: make sure to run `make pyspec` from the root of the specs repository,
+Note: make sure to run `make -B pyspec` from the root of the specs repository,
  to build the parts of the pyspec module derived from the markdown specs.
+The `-B` flag may be helpful to force-overwrite the `pyspec` output after you made a change to the markdown source files.
 
 Run the tests:
 ```