From 4f157ac8d99d0eea4311d7dfccf165d534cbd6e7 Mon Sep 17 00:00:00 2001 From: Chris Elion Date: Mon, 16 Dec 2019 11:05:27 -0800 Subject: [PATCH 1/8] add yaml check --- .pre-commit-config.yaml | 2 ++ config/offline_bc_config.yaml | 4 ++-- config/sac_trainer_config.yaml | 2 -- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a6e56859a6e..2ece27a334e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -55,6 +55,8 @@ repos: types: [markdown] - id: check-merge-conflict args: [--assume-in-merge] + - id: check-yaml + exclude: \.yamato/* - repo: https://github.com/pre-commit/pygrep-hooks rev: v1.4.2 diff --git a/config/offline_bc_config.yaml b/config/offline_bc_config.yaml index 7d071505c53..7f2143c56df 100644 --- a/config/offline_bc_config.yaml +++ b/config/offline_bc_config.yaml @@ -10,7 +10,8 @@ default: num_layers: 2 sequence_length: 32 memory_size: 256 - demo_path: ./UnitySDK/Assets/Demonstrations/.demo + # Change the path here to point to your demo files + demo_path: ./UnitySDK/Assets/Demonstrations/Your_Demo_File.demo FoodCollector: trainer: offline_bc @@ -34,7 +35,6 @@ Hallway: batches_per_epoch: 5 num_layers: 2 hidden_units: 128 - sequence_length: 16 use_recurrent: true memory_size: 256 sequence_length: 32 diff --git a/config/sac_trainer_config.yaml b/config/sac_trainer_config.yaml index 809827854cd..612b7f56754 100644 --- a/config/sac_trainer_config.yaml +++ b/config/sac_trainer_config.yaml @@ -206,7 +206,6 @@ Reacher: summary_freq: 3000 Hallway: - use_recurrent: true sequence_length: 32 num_layers: 2 hidden_units: 128 @@ -218,7 +217,6 @@ Hallway: use_recurrent: true VisualHallway: - use_recurrent: true sequence_length: 32 num_layers: 1 hidden_units: 128 From 0361cf11b5d23184028bfd177ffade4c08a5adb8 Mon Sep 17 00:00:00 2001 From: Chris Elion Date: Mon, 16 Dec 2019 11:19:52 -0800 Subject: [PATCH 2/8] move pre-commit to its own job --- .circleci/config.yml | 44 +++++++++++++++++++++++++++++++++++++------ test_requirements.txt | 1 - 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b1e7ba807bc..3d27255e590 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -64,12 +64,6 @@ jobs: pip freeze > test-reports/pip_versions.txt pytest --cov=ml-agents --cov=ml-agents-envs --cov=gym-unity --cov-report html --junitxml=test-reports/junit.xml -p no:warnings - - run: - name: Check Code Style using pre-commit - command: | - . venv/bin/activate - pre-commit run --show-diff-on-failure --all-files - - run: name: Verify there are no hidden/missing metafiles. # Renaming files or deleting files can leave metafiles behind that makes Unity very unhappy. @@ -88,6 +82,42 @@ jobs: path: htmlcov destination: htmlcov + pre-commit: + parameters: + pyversion: + type: string + description: python version to being used (currently only affects caching). + + docker: + - image: circleci/python:3.7.3 + working_directory: ~/repo/ + + steps: + - checkout + + #- restore_cache: + # keys: + # - v1-precommit-deps-py<< parameters.pyversion >>-{{ checksum ".pre-commit-config.yaml" }} + + - run: + name: Install Dependencies + command: | + python3 -m venv venv + . venv/bin/activate + pip install --upgrade pip + pip install --upgrade setuptools + pip install pre-commit + - + #- save_cache: + # paths: + # - ./venv + # key: v1-precommit-deps-py<< parameters.pyversion >>-{{ checksum ".pre-commit-config.yaml" }} + + - run: + name: Check Code Style using pre-commit + command: | + . venv/bin/activate + pre-commit run --show-diff-on-failure --all-files markdown_link_check: parameters: @@ -241,6 +271,8 @@ workflows: pip_constraints: test_constraints_max_tf2_version.txt - markdown_link_check - protobuf_generation_check + - pre-commit: + pyversion: 3.7.3 - deploy: name: deploy ml-agents-envs directory: ml-agents-envs diff --git a/test_requirements.txt b/test_requirements.txt index 58a3fe39c3e..33b0fbb76b8 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,4 +1,3 @@ # Test-only dependencies should go here, not in setup.py pytest>=3.2.2,<4.0.0 -pre-commit pytest-cov==2.6.1 From cf92681561d0cf546f7bdf26779641bc6f7e1bc2 Mon Sep 17 00:00:00 2001 From: Chris Elion Date: Mon, 16 Dec 2019 11:23:33 -0800 Subject: [PATCH 3/8] fix install --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3d27255e590..7947f8ab281 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -107,7 +107,7 @@ jobs: pip install --upgrade pip pip install --upgrade setuptools pip install pre-commit - - + #- save_cache: # paths: # - ./venv From b48f772d6ee23a9bfafceef7e83057d132c61e3a Mon Sep 17 00:00:00 2001 From: Chris Elion Date: Mon, 16 Dec 2019 11:31:41 -0800 Subject: [PATCH 4/8] try caching --- .circleci/config.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7947f8ab281..b58f666baff 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -95,9 +95,9 @@ jobs: steps: - checkout - #- restore_cache: - # keys: - # - v1-precommit-deps-py<< parameters.pyversion >>-{{ checksum ".pre-commit-config.yaml" }} + - restore_cache: + keys: + - v1-precommit-deps-py<< parameters.pyversion >>-{{ checksum ".pre-commit-config.yaml" }} - run: name: Install Dependencies @@ -108,10 +108,10 @@ jobs: pip install --upgrade setuptools pip install pre-commit - #- save_cache: - # paths: - # - ./venv - # key: v1-precommit-deps-py<< parameters.pyversion >>-{{ checksum ".pre-commit-config.yaml" }} + - save_cache: + paths: + - ~/.cache/pre-commit + key: v1-precommit-deps-py<< parameters.pyversion >>-{{ checksum ".pre-commit-config.yaml" }} - run: name: Check Code Style using pre-commit From 0b79ddf91cfa5d1efb1e0e05abb070d755650a3f Mon Sep 17 00:00:00 2001 From: Chris Elion Date: Mon, 16 Dec 2019 12:23:46 -0800 Subject: [PATCH 5/8] add python version to cache checksum, install before checking --- .circleci/config.yml | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b58f666baff..8b7c8a4c927 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -83,21 +83,23 @@ jobs: destination: htmlcov pre-commit: - parameters: - pyversion: - type: string - description: python version to being used (currently only affects caching). - docker: - image: circleci/python:3.7.3 working_directory: ~/repo/ steps: - checkout + - run: + name: Combine precommit config and python versions for caching + command: | + cat .pre-commit-config.yaml > pre-commit-deps.txt + python -VV >> pre-commit-deps.txt + # TODO REMOVE + echo test1 >> pre-commit-deps.txt - restore_cache: keys: - - v1-precommit-deps-py<< parameters.pyversion >>-{{ checksum ".pre-commit-config.yaml" }} + - v1-precommit-deps-{{ checksum "pre-commit-deps.txt" }} - run: name: Install Dependencies @@ -107,11 +109,13 @@ jobs: pip install --upgrade pip pip install --upgrade setuptools pip install pre-commit + # Install the hooks now so that they'll be cached + pre-commit install-hooks - save_cache: paths: - ~/.cache/pre-commit - key: v1-precommit-deps-py<< parameters.pyversion >>-{{ checksum ".pre-commit-config.yaml" }} + key: v1-precommit-deps-{{ checksum "pre-commit-deps.txt" }} - run: name: Check Code Style using pre-commit @@ -271,8 +275,7 @@ workflows: pip_constraints: test_constraints_max_tf2_version.txt - markdown_link_check - protobuf_generation_check - - pre-commit: - pyversion: 3.7.3 + - pre-commit - deploy: name: deploy ml-agents-envs directory: ml-agents-envs From ee2a00ada0a92552bf4fec51733ff36d2323e9c3 Mon Sep 17 00:00:00 2001 From: Chris Elion Date: Mon, 16 Dec 2019 12:27:46 -0800 Subject: [PATCH 6/8] no-op change --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8b7c8a4c927..10b455036d9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -94,7 +94,7 @@ jobs: command: | cat .pre-commit-config.yaml > pre-commit-deps.txt python -VV >> pre-commit-deps.txt - # TODO REMOVE + # TODO REMOVE - no-op change echo test1 >> pre-commit-deps.txt - restore_cache: From 788c73d54deeb053255d5a11475e2bdbe129fbb7 Mon Sep 17 00:00:00 2001 From: Chris Elion Date: Mon, 16 Dec 2019 12:38:28 -0800 Subject: [PATCH 7/8] cache venv too, whitespace change on precommit --- .circleci/config.yml | 3 +-- .pre-commit-config.yaml | 2 -- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 10b455036d9..04e8c36a77e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -94,8 +94,6 @@ jobs: command: | cat .pre-commit-config.yaml > pre-commit-deps.txt python -VV >> pre-commit-deps.txt - # TODO REMOVE - no-op change - echo test1 >> pre-commit-deps.txt - restore_cache: keys: @@ -115,6 +113,7 @@ jobs: - save_cache: paths: - ~/.cache/pre-commit + - ./venv key: v1-precommit-deps-{{ checksum "pre-commit-deps.txt" }} - run: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2ece27a334e..15054f6a4d1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -63,8 +63,6 @@ repos: hooks: - id: python-check-mock-methods - - - repo: https://github.com/pre-commit/mirrors-pylint rev: v2.4.4 hooks: From db19619775f0ca76c41a3780a4c776763aece141 Mon Sep 17 00:00:00 2001 From: Chris Elion Date: Mon, 16 Dec 2019 12:57:20 -0800 Subject: [PATCH 8/8] add note about yamato yaml --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 15054f6a4d1..1920595894f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -56,6 +56,7 @@ repos: - id: check-merge-conflict args: [--assume-in-merge] - id: check-yaml + # Won't handle the templating in yamato exclude: \.yamato/* - repo: https://github.com/pre-commit/pygrep-hooks