Merge 8032c5e into f6a8e27

.github/pull_request_template.md

@@ -0,0 +1,18 @@
+---
+name: Pull request
+about: Create a pull request for merge
+
+---
+
+## What does this PR do?
+E.g. Describe the added feature or what issue it fixes #(issue)...
+
+## Checklist
+  - [ ] Did you adhere to [PEP-8](https://www.python.org/dev/peps/pep-0008/) standards?
+  - [ ] Did you run black and isort prior to submitting your PR? 
+  - [ ] Does your PR pass all existing unit tests?
+  - [ ] Did you add associated unit tests for any additional functionality?
+  - [ ] Did you provide documentation ([Numpy Docstring format](https://numpydoc.readthedocs.io/en/latest/format.html#style-guide)) whenever possible, even for simple functions or classes?
+
+## Review
+Request will go to reviewers to approve for merge.

.github/workflows/python-coverage.yaml

@@ -21,11 +21,17 @@ jobs:
       with:
         python-version: '3.8'
 
+    - uses: actions/cache@v2
+      with:
+        path: ${{ env.pythonLocation }}
+        key: ${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ hashFiles('REQUIREMENTS.txt') }}-${{ hashFiles('./requirements/DEV_REQUIREMENTS.txt') }}-${{ hashFiles('./requirements/S3_REQUIREMENTS.txt') }}
+
     - name: Install dependencies and dev dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install -r DEV_REQUIREMENTS.txt
-        pip install -r S3_REQUIREMENTS.txt
+        pip install -r REQUIREMENTS.txt
+        pip install -r ./requirements/DEV_REQUIREMENTS.txt
+        pip install -r ./requirements/S3_REQUIREMENTS.txt
 
     - name: Test with pytest
       run: |

.github/workflows/python-docs.yaml

@@ -18,12 +18,18 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: '3.8'
+
+    - uses: actions/cache@v2
+      with:
+        path: ${{ env.pythonLocation }}
+        key: ${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ hashFiles('REQUIREMENTS.txt') }}-${{ hashFiles('./requirements/DEV_REQUIREMENTS.txt') }}-${{ hashFiles('./requirements/S3_REQUIREMENTS.txt') }}
+
     - name: Install dependencies and dev dependencies
       run: |
         python -m pip install --upgrade pip
         pip install -e .[s3]
-        pip install -r DEV_REQUIREMENTS.txt
-        pip install -r S3_REQUIREMENTS.txt
+        pip install -r ./requirements/DEV_REQUIREMENTS.txt
+        pip install -r ./requirements/S3_REQUIREMENTS.txt
 
     - name: Build docs with Portray
       env:

.github/workflows/python-lint.yaml

@@ -0,0 +1,42 @@
+# This workflow will run isort and black linters on PRs
+
+name: lint
+
+# on: workflow_dispatch
+on:
+  pull_request:
+    branches: [master]
+  push:
+    branches: [master]
+
+jobs:
+  run_lint:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.8'
+
+    - uses: actions/cache@v2
+      with:
+        path: ${{ env.pythonLocation }}
+        key: ${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ hashFiles('REQUIREMENTS.txt') }}-${{ hashFiles('./requirements/DEV_REQUIREMENTS.txt') }}-${{ hashFiles('./requirements/S3_REQUIREMENTS.txt') }}
+
+    - name: Install dependencies and dev dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -e .
+        pip install -r ./requirements/DEV_REQUIREMENTS.txt
+        pip install -r ./requirements/S3_REQUIREMENTS.txt
+
+    - name: Run isort linter
+      run: |
+        isort --check . --skip="debug" --skip="versioneer.py" --skip="tests" --skip="_version.py"
+
+    - name: Run black linter
+      run: |
+        black --check . --exclude="versioneer.py|_version.py|debug|tests"

.github/workflows/python-pytest-s3.yaml

@@ -23,12 +23,17 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
 
+    - uses: actions/cache@v2
+      with:
+        path: ${{ env.pythonLocation }}
+        key: ${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ hashFiles('REQUIREMENTS.txt') }}-${{ hashFiles('./requirements/DEV_REQUIREMENTS.txt') }}-${{ hashFiles('./requirements/S3_REQUIREMENTS.txt') }}
+
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
         pip install -e .
-        pip install -r DEV_REQUIREMENTS.txt
-        pip install -r S3_REQUIREMENTS.txt
+        pip install -r ./requirements/DEV_REQUIREMENTS.txt
+        pip install -r ./requirements/S3_REQUIREMENTS.txt
 
     - name: Test with pytest
       run: |

.github/workflows/python-pytest.yml

@@ -23,11 +23,16 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
 
+    - uses: actions/cache@v2
+      with:
+        path: ${{ env.pythonLocation }}
+        key: ${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ hashFiles('REQUIREMENTS.txt') }}-${{ hashFiles('./requirements/DEV_REQUIREMENTS.txt') }}-${{ hashFiles('./requirements/S3_REQUIREMENTS.txt') }}
+
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
         pip install -e .
-        pip install -r DEV_REQUIREMENTS.txt
+        pip install -r ./requirements/DEV_REQUIREMENTS.txt
 
     - name: Test with pytest
       run: |

CONTRIBUTING.md

@@ -4,6 +4,7 @@ Requests in the public repository.
 
 ## Contribution Guidelines
 1. Adhere to [PEP-8](https://www.python.org/dev/peps/pep-0008/) standards.
-2. Any changes to core functionality must pass all existing unit tests.
-3. Additional functionality should have associated unit tests.
-4. Provide documentation (Google Docstring format) whenever possible, even for simple functions or classes.
+2. Run black and isort linters before creating a PR.
+3. Any changes to core functionality must pass all existing unit tests.
+4. Additional functionality should have associated unit tests.
+5. Provide documentation ([Numpy Docstring format](https://numpydoc.readthedocs.io/en/latest/format.html#style-guide)) whenever possible, even for simple functions or classes.

NOTICE.txt

@@ -12,5 +12,5 @@ FMR LLC (https://www.fidelity.com/).
 This product relies on the following works (and the dependencies thereof), installed separately:
 - attrs | https://github.com/python-attrs/attrs | MIT License
 - GitPython | https://github.com/gitpython-developers/GitPython | BSD 3-Clause License
-- PyYAML | https://github.com/yaml/pyyaml | MIT License
-- toml | https://github.com/toml-lang/toml | MIT License
+- pytomlpp | https://github.com/bobfang1992/pytomlpp | MIT License
+- PyYAML | https://github.com/yaml/pyyaml | MIT License

README.md

@@ -3,6 +3,7 @@
 
 [![License](https://img.shields.io/badge/License-Apache%202.0-9cf)](https://opensource.org/licenses/Apache-2.0)
 [![Python](https://img.shields.io/badge/python-3.6+-informational.svg)]()
+[![Style](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
 [![PyPI version](https://badge.fury.io/py/spock-config.svg)](https://badge.fury.io/py/spock-config)
 [![Coverage Status](https://coveralls.io/repos/github/fidelity/spock/badge.svg?branch=master)](https://coveralls.io/github/fidelity/spock?branch=master)
 ![Tests](https://github.com/fidelity/spock/workflows/pytest/badge.svg?branch=master)

REQUIREMENTS.txt

@@ -1,4 +1,4 @@
 attrs
 GitPython
-pyYAML
-toml
+pytomlpp
+pyYAML

docs/Quick-Start.md

@@ -107,7 +107,7 @@ fancier_parameter: 64.64
 most_fancy_parameter: [768, 768, 512, 128]
 ```
 
-Finally, we would run our script and pass the path to the configuration file to the command line (-c or --config):
+Finally, we would run our script and pass the path to the configuration file to the command line (`-c` or `--config`):
 
 ```bash
 $ python simple.py -c simple.yaml
@@ -131,4 +131,16 @@ configuration(s):
     fancy_parameter         float        parameter that multiplies a value 
     fancier_parameter       float        parameter that gets added to product of val and fancy_parameter 
     most_fancy_parameter    List[int]    values to apply basic algebra to 
+```
+
+### Spock As a Drop In For Argparser
+
+`spock` can easily be used as a drop in for argparser. This means that all parameter definitions as required to come in 
+from the command line or from setting defaults within the `@spock` decorated classes. Simply do not pass a `-c` or 
+`--config` argument at the command line and instead pass in all of the automatically generated cmd-line arguments.
+
+
+```bash
+$ python simple.py --BasicConfig.parameter --BasicConfig.fancy_parameter 8.8 --BasicConfig.fancier_parameter 64.64 \
+  --BasicConfig.most_fancy_parameter [768, 768, 512, 128]
 ```

docs/addons/S3.md

@@ -46,11 +46,11 @@ session = boto3.Session(
 
 ### Using the S3Config Object
 
-As an example let's create a basic `@spock` decorated class, instantiate a `S3Config` object from `spock.addons` with
+As an example let's create a basic `@spock` decorated class, instantiate a `S3Config` object from `spock.addons.s3` with
 the `boto3.session.Session` we created above, and pass it to the `ConfigArgBuilder`.
 
 ```python
-from spock.addons import S3Config
+from spock.addons.s3 import S3Config
 from spock.builder import ConfigArgBuilder
 from spock.config import spock
 from typing import List
@@ -123,8 +123,8 @@ With a `S3Config` object passed into the `ConfigArgBuilder` the S3 URI will auto
 If you require any other settings for uploading or downloading files from S3 the `S3Config` class has two extra 
 attributes:
 
-`download_config` which takes a `S3DownloadConfig` object from `spock.addons` which supports all ExtraArgs from
+`download_config` which takes a `S3DownloadConfig` object from `spock.addons.s3` which supports all ExtraArgs from
 [S3Transfer.ALLOWED_DOWNLOAD_ARGS](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/customizations/s3.html#boto3.s3.transfer.S3Transfer.ALLOWED_DOWNLOAD_ARGS)
 
-`upload_config` which takes a `S3UploadConfig` object from `spock.addons` which supports all ExtraArgs from
+`upload_config` which takes a `S3UploadConfig` object from `spock.addons.s3` which supports all ExtraArgs from
 [S3Transfer.ALLOWED_UPLOAD_ARGS](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/customizations/s3.html#boto3.s3.transfer.S3Transfer.ALLOWED_UPLOAD_ARGS)

docs/addons/tuner/Ax.md

@@ -0,0 +1 @@
+Test Placeholder

docs/addons/tuner/Basics.md

@@ -0,0 +1 @@
+Test Placeholder

docs/addons/tuner/Optuna.md

@@ -0,0 +1,5 @@
+Test Placeholder
+
+Ask & Tell Define and Run
+
+https://optuna.readthedocs.io/en/stable/tutorial/20_recipes/009_ask_and_tell.html#

docs/advanced_features/Command-Line-Overrides.md

@@ -117,5 +117,17 @@ We could override the parameters like so (note that the len must match the defin
 
 ```bash
 $ python tutorial.py --config tutorial.yaml --TypeConfig.nested_list.NestedListStuff.one [1,2] \
---TypeConfig.nested_list.NestedListStuff.two [ciao,ciao]
+--TypeConfig.nested_list.NestedListStuff.two ['ciao','ciao']
+```
+
+### Spock As a Drop In For Argparser
+
+`spock` can easily be used as a drop in for argparser. This means that all parameter definitions as required to come in 
+from the command line or from setting defaults within the `@spock` decorated classes. Simply do not pass a `-c` or 
+`--config` argument at the command line and instead pass in all of the automatically generated cmd-line arguments.
+
+
+```bash
+$ python tutorial.py --TypeConfig.nested_list.NestedListStuff.one [1,2] \
+  --TypeConfig.nested_list.NestedListStuff.two [ciao,ciao] ...
 ```

examples/legacy/quick-start/simple.py

@@ -13,7 +13,12 @@ class BasicConfig:
 
 def add_namespace(config):
     # Lets just do some basic algebra here
-    val_sum = sum([(config.fancy_parameter * val) + config.fancier_parameter for val in config.most_fancy_parameter])
+    val_sum = sum(
+        [
+            (config.fancy_parameter * val) + config.fancier_parameter
+            for val in config.most_fancy_parameter
+        ]
+    )
     # If the boolean is true let's round
     if config.parameter:
         val_sum = round(val_sum)
@@ -38,10 +43,14 @@ def main():
     val_sum_namespace = add_namespace(config.BasicConfig)
     print(val_sum_namespace)
     # Or pass by parameter
-    val_sum_parameter = add_by_parameter(config.BasicConfig.fancy_parameter, config.BasicConfig.most_fancy_parameter,
-                                         config.BasicConfig.fancier_parameter, config.BasicConfig.parameter)
+    val_sum_parameter = add_by_parameter(
+        config.BasicConfig.fancy_parameter,
+        config.BasicConfig.most_fancy_parameter,
+        config.BasicConfig.fancier_parameter,
+        config.BasicConfig.parameter,
+    )
     print(val_sum_parameter)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()

examples/legacy/tutorial/advanced/basic_nn.py

@@ -6,12 +6,16 @@ class BasicNet(nn.Module):
     def __init__(self, model_config):
         super(BasicNet, self).__init__()
         # Make a dictionary of activation functions to select from
-        self.act_fncs = {'relu': nn.ReLU, 'gelu': nn.GELU, 'tanh': nn.Tanh}
+        self.act_fncs = {"relu": nn.ReLU, "gelu": nn.GELU, "tanh": nn.Tanh}
         self.use_act = self.act_fncs.get(model_config.activation)()
         # Define the layers manually (avoiding list comprehension for clarity)
         self.layer_1 = nn.Linear(model_config.n_features, model_config.hidden_sizes[0])
-        self.layer_2 = nn.Linear(model_config.hidden_sizes[0], model_config.hidden_sizes[1])
-        self.layer_3 = nn.Linear(model_config.hidden_sizes[1], model_config.hidden_sizes[2])
+        self.layer_2 = nn.Linear(
+            model_config.hidden_sizes[0], model_config.hidden_sizes[1]
+        )
+        self.layer_3 = nn.Linear(
+            model_config.hidden_sizes[1], model_config.hidden_sizes[2]
+        )
         # Define some dropout layers
         self.dropout = []
         if model_config.dropout is not None: