S3 functionality unit tests. Reorg of all other unit tests for sanity…

… purposes.

Signed-off-by: Nicholas Cilfone <nicholas.cilfone@fmr.com>

.github/workflows/python-coverage.yaml

@@ -11,7 +11,6 @@ on:
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
 
     steps:
@@ -26,6 +25,7 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install -r DEV_REQUIREMENTS.txt
+        pip install -r S3_REQUIREMENTS.txt
 
     - name: Test with pytest
       run: |

.github/workflows/python-pytest-s3.yaml

@@ -0,0 +1,43 @@
+# This workflow will install Python dependencies, run S3 tests with a variety of Python versions
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: pytest-s3
+
+on:
+  pull_request:
+    branches: [master]
+  push:
+    branches: [master]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.6, 3.7, 3.8, 3.9]
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -e .
+        pip install -r DEV_REQUIREMENTS.txt
+        pip install -r S3_REQUIREMENTS.txt
+
+    - name: Test with pytest
+      run: |
+        pytest tests/s3 --cov=spock --cov-config=.coveragerc --junitxml=junit/test-results-${{ matrix.python-version }}.xml --cov-report=xml --cov-report=html
+
+    - name: Upload pytest test results
+      uses: actions/upload-artifact@v2
+      with:
+        name: pytest-results-${{ matrix.python-version }}
+        path: junit/test-results-${{ matrix.python-version }}.xml
+      # Use always() to always run this step to publish test results when there are test failures
+      if: ${{ always() }}

.github/workflows/python-pytest.yml

@@ -1,4 +1,4 @@
-# This workflow will install Python dependencies, run tests with a variety of Python versions
+# This workflow will install Python dependencies, run general tests with a variety of Python versions
 # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
 
 name: pytest
@@ -11,7 +11,6 @@ on:
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -23,15 +22,17 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: ${{ matrix.python-version }}
+
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
         pip install -e .
-        pip install pylint pytest pytest-cov
-        if [ -f REQUIREMENTS.txt ]; then pip install -r REQUIREMENTS.txt; fi
+        pip install -r DEV_REQUIREMENTS.txt
+
     - name: Test with pytest
       run: |
-        pytest --cov=spock --cov-config=.coveragerc --junitxml=junit/test-results-${{ matrix.python-version }}.xml --cov-report=xml --cov-report=html
+        pytest tests/base --cov=spock --cov-config=.coveragerc --junitxml=junit/test-results-${{ matrix.python-version }}.xml --cov-report=xml --cov-report=html
+
     - name: Upload pytest test results
       uses: actions/upload-artifact@v2
       with:

DEV_REQUIREMENTS.txt

@@ -1,6 +1,7 @@
 -r REQUIREMENTS.txt
 coveralls
 coverage
+moto
 portray
 pytest
 pytest-cov

S3_REQUIREMENTS.txt

@@ -0,0 +1,4 @@
+boto3
+botocore
+hurry.filesize
+s3transfer

setup.py

@@ -15,6 +15,9 @@
 with open('REQUIREMENTS.txt', 'r') as fid:
     install_reqs = [str(req) for req in parse_requirements(fid)]
 
+with open('S3_REQUIREMENTS.txt', 'r') as fid:
+    s3_reqs = [str(req) for req in parse_requirements(fid)]
+
 setuptools.setup(
     name='spock-config',
     description='Spock is a framework designed to help manage complex parameter configurations for Python applications',
@@ -49,5 +52,5 @@
     packages=setuptools.find_packages(exclude=["*.tests", "*.tests.*", "tests.*", "tests"]),
     python_requires='>=3.6',
     install_requires=install_reqs,
-    extras_require={'s3': ['boto3', 'botocore', 's3transfer', 'hurry.filesize']}
+    extras_require={'s3': s3_reqs}
 )

spock/addons/s3/configs.py

@@ -49,16 +49,20 @@ class S3Config:
     *Attributes*:
 
         session: instantiated boto3 session object
-        s3_session: automatically generated s3 client from the boto3 session
+        s3_session: automatically generated s3 client from the boto3 session if not provided
         kms_arn: AWS KMS key ARN (optional)
         temp_folder: temporary working folder to write/read spock configuration(s) (optional: defaults to /tmp)
+        download_config: S3DownloadConfig for extra download configs (optional)
+        upload_config: S3UploadConfig for extra upload configs (optional)
 
     """
     session: boto3.Session
-    s3_session: BaseClient = attr.ib(init=False)
+    # s3_session: BaseClient = attr.ib(init=False)
+    s3_session: typing.Optional[BaseClient] = None
     temp_folder: typing.Optional[str] = '/tmp/'
     download_config: S3DownloadConfig = S3DownloadConfig()
     upload_config: S3UploadConfig = S3UploadConfig()
 
     def __attrs_post_init__(self):
-        self.s3_session = self.session.client('s3')
+        if self.s3_session is None:
+            self.s3_session = self.session.client('s3')

spock/handlers.py

@@ -148,6 +148,8 @@ def _handle_possible_s3_save_path(path: str, name: str, create_path: bool,
         """
         is_s3 = check_path_s3(path=path)
         if is_s3:
+            if s3_config is None:
+                raise ValueError('Save to S3 -- Missing S3Config object which is necessary to handle S3 style paths')
             write_path = f'{s3_config.temp_folder}/{name}'
             # Strip double slashes if exist
             write_path = write_path.replace(r'//', r'/')