From c8d288b20be936e41f473ba24f8b450a1c137eea Mon Sep 17 00:00:00 2001 From: Nate Prewitt Date: Sat, 4 Sep 2021 21:40:05 -0700 Subject: [PATCH 1/4] Add pre-commit config --- .github/workflows/lint.yml | 19 +++++++++++++++++++ .github/workflows/run-tests.yml | 1 - .pre-commit-config.yaml | 12 ++++++++++++ setup.cfg | 8 ++++++++ 4 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/lint.yml create mode 100644 .pre-commit-config.yaml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000000..010726887b --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,19 @@ +name: Lint code + +on: + push: + pull_request: + branches-ignore: [ master ] + +jobs: + lint: + runs-on: ubuntu-20.04 + + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.9 + uses: actions/setup-python@v2 + with: + python-version: 3.9 + - name: Run pre-commit + uses: pre-commit/action@v2.0.0 diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 5f51b5a496..cc95ba6448 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -1,4 +1,3 @@ - name: Run tests on: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000..e53d776a52 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,12 @@ +exclude: ^(.github|.changes|docs/|boto3/compat.py|boto3/data) +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.3.0 + hooks: + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace +- repo: https://github.com/pycqa/flake8 + rev: 3.9.2 + hooks: + - id: flake8 diff --git a/setup.cfg b/setup.cfg index c0213ef94c..eab09c1966 100644 --- a/setup.cfg +++ b/setup.cfg @@ -9,3 +9,11 @@ requires_dist = [options.extras_require] crt = botocore[crt]>=1.21.0,<2.0a0 + +[flake8] +ignore = E203,E501,W503,W504 +exclude = + docs, + boto3/compat.py, + boto3/data, + .changes From bbfbde9e4263205a5878141b35fbf97609132296 Mon Sep 17 00:00:00 2001 From: Nate Prewitt Date: Sat, 4 Sep 2021 21:55:18 -0700 Subject: [PATCH 2/4] Fix flake8 conflicts --- CHANGELOG.rst | 1 - CODE_OF_CONDUCT.md | 1 - README.rst | 7 +- boto3/compat.py | 2 +- boto3/docs/collection.py | 8 +- boto3/dynamodb/conditions.py | 5 +- scripts/ci/run-crt-tests | 2 +- scripts/new-change | 2 +- tests/__init__.py | 12 --- tests/functional/docs/test_ec2.py | 22 ++++-- tests/functional/docs/test_smoke.py | 15 ++-- .../dynamodb/test_stubber_conditions.py | 21 +++-- tests/functional/dynamodb/test_table.py | 2 +- tests/functional/test_s3.py | 2 +- tests/functional/test_smoke.py | 2 + tests/integration/test_dynamodb.py | 12 ++- tests/integration/test_s3.py | 13 +-- tests/integration/test_session.py | 2 +- tests/unit/docs/test_collection.py | 10 +-- tests/unit/dynamodb/test_table.py | 79 ++++++++++++------- tests/unit/dynamodb/test_transform.py | 17 ++-- tests/unit/dynamodb/test_types.py | 30 +++---- tests/unit/resources/test_collection.py | 5 +- tests/unit/resources/test_collection_smoke.py | 2 + tests/unit/resources/test_factory.py | 29 ++++--- tests/unit/resources/test_model.py | 29 +++++-- tests/unit/resources/test_params.py | 8 +- tests/unit/resources/test_response.py | 7 +- tests/unit/s3/test_inject.py | 1 + tests/unit/test_session.py | 8 +- 30 files changed, 208 insertions(+), 148 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e8f2d514c6..bc327b2ff7 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8157,4 +8157,3 @@ CHANGELOG * feature:Resources: Supports S3, EC2, SQS, SNS, and IAM resources * feature:Clients: Supports low-level clients for most services - diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 826378380e..5b627cfa60 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -2,4 +2,3 @@ This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact opensource-codeofconduct@amazon.com with any additional questions or comments. - diff --git a/README.rst b/README.rst index b7ba1ab9ef..d7649b6a95 100644 --- a/README.rst +++ b/README.rst @@ -49,10 +49,10 @@ Assuming that you have Python and ``virtualenv`` installed, set up your environm $ python -m pip install boto3 - + Using Boto3 ~~~~~~~~~~~~~~ -After installing boto3 +After installing boto3 Next, set up credentials (in e.g. ``~/.aws/credentials``): @@ -68,7 +68,7 @@ Then, set up a default region (in e.g. ``~/.aws/config``): [default] region=us-east-1 - + Other credentials configuration method can be found `here `__ Then, from a Python interpreter: @@ -137,4 +137,3 @@ More Resources * `NOTICE `__ * `Changelog `__ * `License `__ - diff --git a/boto3/compat.py b/boto3/compat.py index 099fcdedc7..7b617bbf56 100644 --- a/boto3/compat.py +++ b/boto3/compat.py @@ -75,7 +75,7 @@ def _warn_deprecated_python(): 'aws-cli-v1/' } deprecated_versions = { - (2,7): py_27_params, + (2, 7): py_27_params, } py_version = sys.version_info[:2] diff --git a/boto3/docs/collection.py b/boto3/docs/collection.py index 3f7a38f46b..db0fb799a2 100644 --- a/boto3/docs/collection.py +++ b/boto3/docs/collection.py @@ -185,11 +185,13 @@ def document_collection_method(section, resource_name, action_name, 'method_description': ( 'Creates an iterable of all %s resources ' 'in the collection filtered by kwargs passed to ' - 'method.' % collection_model.resource.type + - 'A %s collection will include all resources by ' + 'method. A %s collection will include all resources by ' 'default if no filters are provided, and extreme ' 'caution should be taken when performing actions ' - 'on all resources.'% collection_model.resource.type), + 'on all resources.' % ( + collection_model.resource.type, + collection_model.resource.type + )), 'example_prefix': '%s_iterator = %s.%s.filter' % ( xform_name(collection_model.resource.type), example_resource_name, collection_model.name), diff --git a/boto3/dynamodb/conditions.py b/boto3/dynamodb/conditions.py index c63bd2a5c4..41850b14b7 100644 --- a/boto3/dynamodb/conditions.py +++ b/boto3/dynamodb/conditions.py @@ -144,8 +144,9 @@ def __init__(self, *values): AttributeBase.__init__(self, values[0].name) def __eq__(self, other): - return ConditionBase.__eq__(self, other) and \ - AttributeBase.__eq__(self, other) + return ( + ConditionBase.__eq__(self, other) and AttributeBase.__eq__(self, other) + ) def __ne__(self, other): return not self.__eq__(other) diff --git a/scripts/ci/run-crt-tests b/scripts/ci/run-crt-tests index 63fa6f8179..9ff2f067a6 100755 --- a/scripts/ci/run-crt-tests +++ b/scripts/ci/run-crt-tests @@ -29,7 +29,7 @@ def run(command): try: - import awscrt + import awscrt # noqa except ImportError: print("MISSING DEPENDENCY: awscrt must be installed to run the crt tests.") sys.exit(1) diff --git a/scripts/new-change b/scripts/new-change index 73bd9ab26d..4d0cbcdfc5 100755 --- a/scripts/new-change +++ b/scripts/new-change @@ -135,7 +135,7 @@ def replace_issue_references(parsed, repo_name): '`%s `__' % ( match.group(), repo_name, number)) - new_description = re.sub('#\d+', linkify, description) + new_description = re.sub(r'#\d+', linkify, description) parsed['description'] = new_description diff --git a/tests/__init__.py b/tests/__init__.py index 64b56c40c8..169862aac3 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -12,24 +12,12 @@ # language governing permissions and limitations under the License. import random -import sys import time -from botocore.compat import six - - import unittest from unittest import mock -# In python 3, order matters when calling assertEqual to -# compare lists and dictionaries with lists. Therefore, -# assertItemsEqual needs to be used but it is renamed to -# assertCountEqual in python 3. -if six.PY2: - unittest.TestCase.assertCountEqual = unittest.TestCase.assertItemsEqual - - def unique_id(name): """ Generate a unique ID that includes the given name, diff --git a/tests/functional/docs/test_ec2.py b/tests/functional/docs/test_ec2.py index 6f5cda8f99..979b038665 100644 --- a/tests/functional/docs/test_ec2.py +++ b/tests/functional/docs/test_ec2.py @@ -19,17 +19,23 @@ class TestInstanceDeleteTags(BaseDocsFunctionalTests): def setUp(self): self.documenter = ServiceDocumenter( - 'ec2', session=Session(region_name='us-east-1')) + 'ec2', session=Session(region_name='us-east-1') + ) self.generated_contents = self.documenter.document_service() self.generated_contents = self.generated_contents.decode('utf-8') def test_delete_tags_method_is_documented(self): contents = self.get_class_document_block( - 'EC2.Instance', self.generated_contents) + 'EC2.Instance', self.generated_contents + ) method_contents = self.get_method_document_block( - 'delete_tags', contents) - self.assert_contains_lines_in_order([ - 'response = instance.delete_tags(', - 'DryRun=True|False,', - 'Tags=[', - ], method_contents) + 'delete_tags', contents + ) + self.assert_contains_lines_in_order( + [ + 'response = instance.delete_tags(', + 'DryRun=True|False,', + 'Tags=[', + ], + method_contents + ) diff --git a/tests/functional/docs/test_smoke.py b/tests/functional/docs/test_smoke.py index 34483b93d6..c4ea19ef2f 100644 --- a/tests/functional/docs/test_smoke.py +++ b/tests/functional/docs/test_smoke.py @@ -23,12 +23,13 @@ def botocore_session(): return botocore.session.get_session() + @pytest.fixture def boto3_session(): return boto3.Session(region_name='us-east-1') + def all_services(): - botocore_session = botocore.session.get_session() session = boto3.Session(region_name='us-east-1') for service_name in session.get_available_services(): yield service_name @@ -52,15 +53,13 @@ def test_documentation( # Check that all of the services have the appropriate title _assert_has_title(generated_docs, client) - # Check that all services have the client documented. _assert_has_client_documentation(generated_docs, service_name, client) - - #If the service has resources, make sure the service resource - #is at least documented. + # If the service has resources, make sure the service resource + # is at least documented. if service_name in available_resources: - + resource = boto3.resource(service_name, 'us-east-1') _assert_has_resource_documentation( generated_docs, service_name, resource @@ -68,8 +67,7 @@ def test_documentation( # If the client can paginate, make sure the paginators are documented. try: - paginator_model = botocore_session.get_paginator_model( - service_name) + paginator_model = botocore_session.get_paginator_model(service_name) _assert_has_paginator_documentation( generated_docs, service_name, client, sorted(paginator_model._paginator_config) @@ -77,7 +75,6 @@ def test_documentation( except DataNotFoundError: pass - # If the client has waiters, make sure the waiters are documented. if client.waiter_names: waiter_model = botocore_session.get_waiter_model(service_name) diff --git a/tests/functional/dynamodb/test_stubber_conditions.py b/tests/functional/dynamodb/test_stubber_conditions.py index b326896588..3139c61972 100644 --- a/tests/functional/dynamodb/test_stubber_conditions.py +++ b/tests/functional/dynamodb/test_stubber_conditions.py @@ -31,11 +31,15 @@ def test_table_query_can_be_stubbed_with_expressions(self): ) stubber = Stubber(table.meta.client) - stubber.add_response('query', dict(Items=list()), expected_params=dict( + stubber.add_response( + 'query', + dict(Items=list()), + expected_params=dict( TableName='mytable', KeyConditionExpression=key_expr, FilterExpression=filter_expr - )) + ) + ) with stubber: response = table.query(KeyConditionExpression=key_expr, @@ -46,15 +50,20 @@ def test_table_query_can_be_stubbed_with_expressions(self): def test_table_scan_can_be_stubbed_with_expressions(self): table = self.resource.Table('mytable') - filter_expr = Attr('myattr').eq('foo') & ( - Attr('myattr2').lte('buzz') | Attr('myattr2').gte('fizz') + filter_expr = ( + Attr('myattr').eq('foo') & + (Attr('myattr2').lte('buzz') | Attr('myattr2').gte('fizz')) ) stubber = Stubber(table.meta.client) - stubber.add_response('scan', dict(Items=list()), expected_params=dict( + stubber.add_response( + 'scan', + dict(Items=list()), + expected_params=dict( TableName='mytable', FilterExpression=filter_expr - )) + ) + ) with stubber: response = table.scan(FilterExpression=filter_expr) diff --git a/tests/functional/dynamodb/test_table.py b/tests/functional/dynamodb/test_table.py index 5de09728f6..b96bfe8661 100644 --- a/tests/functional/dynamodb/test_table.py +++ b/tests/functional/dynamodb/test_table.py @@ -10,7 +10,7 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -from tests import unittest, mock +from tests import unittest import boto3 from botocore.stub import Stubber diff --git a/tests/functional/test_s3.py b/tests/functional/test_s3.py index 0b248ccf5d..a7d8d7901d 100644 --- a/tests/functional/test_s3.py +++ b/tests/functional/test_s3.py @@ -377,7 +377,7 @@ def stub_get_object(self, full_contents, start_byte=0, end_byte=None): # If this is a ranged get, ContentRange needs to be returned, # contents needs to be pruned, and Range needs to be an expected param. if end_byte is not None: - contents = full_contents[start_byte:end_byte+1] + contents = full_contents[start_byte:end_byte + 1] part_range = 'bytes=%s-%s' % (start_byte, end_byte_range) content_range = 'bytes=%s-%s/%s' % ( start_byte, end_byte, len(full_contents)) diff --git a/tests/functional/test_smoke.py b/tests/functional/test_smoke.py index 19c13da40e..12a05ffe8b 100644 --- a/tests/functional/test_smoke.py +++ b/tests/functional/test_smoke.py @@ -17,6 +17,7 @@ boto3_session = None + def create_session(): global boto3_session if boto3_session is None: @@ -28,6 +29,7 @@ def create_session(): return boto3_session + def _all_resources(): session = create_session() for service_name in session.get_available_resources(): diff --git a/tests/integration/test_dynamodb.py b/tests/integration/test_dynamodb.py index df46b55a06..2453322472 100644 --- a/tests/integration/test_dynamodb.py +++ b/tests/integration/test_dynamodb.py @@ -171,16 +171,20 @@ def test_condition_attribute_type(self): def test_condition_and(self): r = self.scan( - filter_expression=(Attr('MyHashKey').eq('mykey') & - Attr('MyString').eq('mystring'))) + filter_expression=( + Attr('MyHashKey').eq('mykey') & Attr('MyString').eq('mystring') + ) + ) item = r['Items'][0] self.assertTrue( item['MyHashKey'] == 'mykey' and item['MyString'] == 'mystring') def test_condition_or(self): r = self.scan( - filter_expression=(Attr('MyHashKey').eq('mykey2') | - Attr('MyString').eq('mystring'))) + filter_expression=( + Attr('MyHashKey').eq('mykey2') | Attr('MyString').eq('mystring') + ) + ) item = r['Items'][0] self.assertTrue( item['MyHashKey'] == 'mykey2' or item['MyString'] == 'mystring') diff --git a/tests/integration/test_s3.py b/tests/integration/test_s3.py index 30a5569c49..fa5191bde7 100644 --- a/tests/integration/test_s3.py +++ b/tests/integration/test_s3.py @@ -472,6 +472,7 @@ def test_callback_called_once_with_sigv4(self): # twice when using signature version 4. self.amount_seen = 0 lock = threading.Lock() + def progress_callback(amount): with lock: self.amount_seen += amount @@ -593,9 +594,11 @@ def test_download_above_threshold(self): def test_download_file_with_directory_not_exist(self): transfer = self.create_s3_transfer() - self.client.put_object(Bucket=self.bucket_name, - Key='foo.txt', - Body=b'foo') + self.client.put_object( + Bucket=self.bucket_name, + Key='foo.txt', + Body=b'foo' + ) self.addCleanup(self.delete_object, 'foo.txt') download_path = os.path.join(self.files.rootdir, 'a', 'b', 'c', 'downloaded.txt') @@ -667,7 +670,7 @@ def test_transfer_methods_through_bucket(self): # This is just a sanity check to ensure that the bucket interface work. key = 'bucket.txt' bucket = self.session.resource('s3').Bucket(self.bucket_name) - filename = self.files.create_file_with_size(key, 1024*1024) + filename = self.files.create_file_with_size(key, 1024 * 1024) bucket.upload_file(Filename=filename, Key=key) self.addCleanup(self.delete_object, key) download_path = os.path.join(self.files.rootdir, unique_id('foo')) @@ -678,7 +681,7 @@ def test_transfer_methods_through_object(self): # This is just a sanity check to ensure that the object interface work. key = 'object.txt' obj = self.session.resource('s3').Object(self.bucket_name, key) - filename = self.files.create_file_with_size(key, 1024*1024) + filename = self.files.create_file_with_size(key, 1024 * 1024) obj.upload_file(Filename=filename) self.addCleanup(self.delete_object, key) download_path = os.path.join(self.files.rootdir, unique_id('foo')) diff --git a/tests/integration/test_session.py b/tests/integration/test_session.py index fb435732a0..445c16fa1b 100644 --- a/tests/integration/test_session.py +++ b/tests/integration/test_session.py @@ -10,7 +10,7 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -from tests import unittest, unique_id +from tests import unittest import botocore.session import boto3.session diff --git a/tests/unit/docs/test_collection.py b/tests/unit/docs/test_collection.py index 5ebb82aac3..353b19e2e1 100644 --- a/tests/unit/docs/test_collection.py +++ b/tests/unit/docs/test_collection.py @@ -22,9 +22,9 @@ def test_document_collections(self): self.assert_contains_lines_in_order([ '.. py:attribute:: samples', ' A collection of Sample resources.' - 'A Sample Collection will include all resources by default, ' - 'and extreme caution should be taken when performing actions ' - 'on all resources.', + 'A Sample Collection will include all resources by default, ' + 'and extreme caution should be taken when performing actions ' + 'on all resources.', ' .. py:method:: all()', (' Creates an iterable of all Sample resources in the ' 'collection.'), @@ -35,7 +35,7 @@ def test_document_collections(self): ' :returns: A list of Sample resources', ' .. py:method:: filter(**kwargs)', (' Creates an iterable of all Sample resources in ' - 'the collection filtered by kwargs passed to method.' + 'the collection filtered by kwargs passed to method. ' 'A Sample collection will include all resources by default ' 'if no filters are provided, and extreme caution should be ' 'taken when performing actions on all resources'), @@ -102,4 +102,4 @@ def test_document_collections(self): ' :rtype: list(:py:class:`myservice.Sample`)', ' :returns: A list of Sample resources', ' ' - ]) \ No newline at end of file + ]) diff --git a/tests/unit/dynamodb/test_table.py b/tests/unit/dynamodb/test_table.py index 6fdb728d34..d7386a819e 100644 --- a/tests/unit/dynamodb/test_table.py +++ b/tests/unit/dynamodb/test_table.py @@ -225,7 +225,6 @@ def test_never_send_more_than_max_batch_size(self): self.assert_batch_write_calls_are([first_batch, second_batch, third_batch]) - def test_repeated_flushing_on_exit(self): # We're going to simulate unprocessed_items # returning multiple unprocessed items across calls. @@ -348,39 +347,63 @@ def test_auto_dedup_for_dup_requests(self): first_batch = { 'RequestItems': { self.table_name: [ - {'PutRequest': { 'Item': { - 'pkey': 'foo1', - 'skey': 'bar1', - 'other': 'other2' - }}}, - {'PutRequest': { 'Item': { - 'pkey': 'foo1', - 'skey': 'bar2', - 'other': 'other3' - }}}, - {'DeleteRequest': {'Key': { - 'pkey': 'foo2', - 'skey': 'bar2', - }}}, - {'DeleteRequest': {'Key': { - 'pkey': 'foo2', - 'skey': 'bar3', - }}}, - {'DeleteRequest': {'Key': { - 'pkey': 'foo3', - 'skey': 'bar3', - }}}, + { + 'PutRequest': { + 'Item': { + 'pkey': 'foo1', + 'skey': 'bar1', + 'other': 'other2' + } + } + }, + { + 'PutRequest': { + 'Item': { + 'pkey': 'foo1', + 'skey': 'bar2', + 'other': 'other3' + } + } + }, + { + 'DeleteRequest': { + 'Key': { + 'pkey': 'foo2', + 'skey': 'bar2', + } + } + }, + { + 'DeleteRequest': { + 'Key': { + 'pkey': 'foo2', + 'skey': 'bar3', + } + } + }, + { + 'DeleteRequest': { + 'Key': { + 'pkey': 'foo3', + 'skey': 'bar3', + } + } + }, ] } } second_batch = { 'RequestItems': { self.table_name: [ - {'PutRequest': { 'Item': { - 'pkey': 'foo1', - 'skey': 'bar1', - 'other': 'other2' - }}}, + { + 'PutRequest': { + 'Item': { + 'pkey': 'foo1', + 'skey': 'bar1', + 'other': 'other2' + } + } + }, ] } } diff --git a/tests/unit/dynamodb/test_transform.py b/tests/unit/dynamodb/test_transform.py index aecbbefb8a..8dec344d33 100644 --- a/tests/unit/dynamodb/test_transform.py +++ b/tests/unit/dynamodb/test_transform.py @@ -135,9 +135,9 @@ def test_transform_map(self): transformation=self.transformation, target_shape=self.target_shape) assert input_params == { - 'TransformMe': - {'foo': self.transformed_value}, - 'LeaveAlone': {'foo': self.original_value}} + 'TransformMe': {'foo': self.transformed_value}, + 'LeaveAlone': {'foo': self.original_value} + } def test_transform_list(self): input_params = { @@ -315,10 +315,13 @@ def test_transform_nested_list(self): transformation=self.transformation, target_shape=self.target_shape) assert input_params == { - 'TargetedWrapperList': [[ - self.transformed_value, self.transformed_value]], - 'UntargetedWrapperList': [[ - self.original_value, self.original_value]]} + 'TargetedWrapperList': [ + [self.transformed_value, self.transformed_value] + ], + 'UntargetedWrapperList': [ + [self.original_value, self.original_value] + ] + } def test_transform_incorrect_type_for_structure(self): input_params = { diff --git a/tests/unit/dynamodb/test_types.py b/tests/unit/dynamodb/test_types.py index d7545476ff..347f27016f 100644 --- a/tests/unit/dynamodb/test_types.py +++ b/tests/unit/dynamodb/test_types.py @@ -16,8 +16,6 @@ from tests import unittest -from botocore.compat import six - from boto3.dynamodb.types import Binary, TypeSerializer, TypeDeserializer @@ -97,8 +95,6 @@ def test_serialize_binary(self): def test_serialize_bytearray(self): assert self.serializer.serialize(bytearray([1])) == {'B': b'\x01'} - @pytest.mark.skipif(six.PY2, - reason='This is a test when using python3 version of bytes') def test_serialize_bytes(self): assert self.serializer.serialize(b'\x01') == {'B': b'\x01'} @@ -141,9 +137,13 @@ def test_serialize_tuple(self): def test_serialize_map(self): serialized_value = self.serializer.serialize( - {'foo': 'bar', 'baz': {'biz': 1}}) - assert serialized_value == {'M': - {'foo': {'S': 'bar'}, 'baz': {'M': {'biz': {'N': '1'}}}} + {'foo': 'bar', 'baz': {'biz': 1}} + ) + assert serialized_value == { + 'M': { + 'foo': {'S': 'bar'}, + 'baz': {'M': {'biz': {'N': '1'}}} + } } @@ -179,22 +179,22 @@ def test_deserialize_binary(self): def test_deserialize_number_set(self): assert self.deserializer.deserialize( - {'NS': ['1', '1.25']}), set([Decimal('1') == Decimal('1.25')]) + {'NS': ['1', '1.25']}) == set([Decimal('1'), Decimal('1.25')]) def test_deserialize_string_set(self): assert self.deserializer.deserialize( - {'SS': ['foo', 'bar']}) == set(['foo', 'bar']) + {'SS': ['foo', 'bar']}) == set(['foo', 'bar']) def test_deserialize_binary_set(self): - assert self.deserializer.deserialize({'BS': [b'\x00', b'\x01']}) == set( - [Binary(b'\x00'), Binary(b'\x01')]) + assert self.deserializer.deserialize( + {'BS': [b'\x00', b'\x01']}) == set([Binary(b'\x00'), Binary(b'\x01')]) def test_deserialize_list(self): - assert self.deserializer.deserialize({'L': - [{'N': '1'}, {'S': 'foo'}, {'L': [{'N': '1.25'}]}]} + assert self.deserializer.deserialize( + {'L': [{'N': '1'}, {'S': 'foo'}, {'L': [{'N': '1.25'}]}]} ) == [Decimal('1'), 'foo', [Decimal('1.25')]] def test_deserialize_map(self): - assert self.deserializer.deserialize({'M': {'foo': - {'S': 'mystring'}, 'bar': {'M': {'baz': {'N': '1'}}}}} + assert self.deserializer.deserialize( + {'M': {'foo': {'S': 'mystring'}, 'bar': {'M': {'baz': {'N': '1'}}}}} ) == {'foo': 'mystring', 'bar': {'baz': Decimal('1')}} diff --git a/tests/unit/resources/test_collection.py b/tests/unit/resources/test_collection.py index 83e48a5d13..384989d163 100644 --- a/tests/unit/resources/test_collection.py +++ b/tests/unit/resources/test_collection.py @@ -16,8 +16,9 @@ from botocore.model import ServiceModel from boto3.utils import ServiceContext -from boto3.resources.collection import CollectionFactory, CollectionManager, \ - ResourceCollection +from boto3.resources.collection import ( + CollectionFactory, CollectionManager, ResourceCollection +) from boto3.resources.base import ResourceMeta from boto3.resources.factory import ResourceFactory from boto3.resources.model import Collection diff --git a/tests/unit/resources/test_collection_smoke.py b/tests/unit/resources/test_collection_smoke.py index fc1a5d6ad2..798a2bd2b0 100644 --- a/tests/unit/resources/test_collection_smoke.py +++ b/tests/unit/resources/test_collection_smoke.py @@ -87,6 +87,7 @@ def _collection_test_args(): for collection_model in resource_model.collections: yield (client, service_name, resource_name, collection_model) + @pytest.mark.parametrize( 'collection_args', _collection_test_args() @@ -100,6 +101,7 @@ def test_all_collections_have_paginators_if_needed(collection_args): # should be a paginator applied to it. _assert_collection_has_paginator_if_needed(*collection_args) + def _assert_collection_has_paginator_if_needed( client, service_name, resource_name, collection_model ): diff --git a/tests/unit/resources/test_factory.py b/tests/unit/resources/test_factory.py index 5eb25c34a6..0a675be22f 100644 --- a/tests/unit/resources/test_factory.py +++ b/tests/unit/resources/test_factory.py @@ -20,7 +20,6 @@ from boto3.resources.base import ServiceResource from boto3.resources.collection import CollectionManager from boto3.resources.factory import ResourceFactory -from boto3.resources.action import WaiterAction class BaseTestResourceFactory(BaseTestCase): @@ -35,13 +34,13 @@ def load(self, resource_name, resource_json_definition=None, resource_json_definition = {} if resource_json_definitions is None: resource_json_definitions = {} - service_context=ServiceContext( + service_context = ServiceContext( service_name='test', resource_json_definitions=resource_json_definitions, service_model=service_model, service_waiter_model=None ) - + return self.factory.load_from_definition( resource_name=resource_name, single_resource_json_definition=resource_json_definition, @@ -573,10 +572,14 @@ def test_resource_loads_waiters(self): model = { "waiters": { "Exists": { - "waiterName": "BucketExists", - "params": [ - {"target": "Bucket", "source": "identifier", - "name": "Name"}] + "waiterName": "BucketExists", + "params": [ + { + "target": "Bucket", + "source": "identifier", + "name": "Name" + } + ] } } } @@ -595,10 +598,14 @@ def test_resource_waiter_calls_waiter_method(self, waiter_action_cls): model = { "waiters": { "Exists": { - "waiterName": "BucketExists", - "params": [ - {"target": "Bucket", "source": "identifier", - "name": "Name"}] + "waiterName": "BucketExists", + "params": [ + { + "target": "Bucket", + "source": "identifier", + "name": "Name" + } + ] } } } diff --git a/tests/unit/resources/test_model.py b/tests/unit/resources/test_model.py index de98860d2e..c49f29f954 100644 --- a/tests/unit/resources/test_model.py +++ b/tests/unit/resources/test_model.py @@ -169,8 +169,11 @@ def test_resource_references(self): 'resource': { 'type': 'Frob', 'identifiers': [ - {'target':'Id', 'source':'data', - 'path':'FrobId'} + { + 'target': 'Id', + 'source': 'data', + 'path': 'FrobId' + } ] } } @@ -237,6 +240,7 @@ def test_waiter(self): assert waiter.waiter_name == 'ObjectExists' assert waiter.params[0].target == 'Bucket' + class TestRenaming(BaseTestCase): def test_multiple(self): # This tests a bunch of different renames working together @@ -250,8 +254,11 @@ def test_multiple(self): 'resource': { 'type': 'Frob', 'identifiers': [ - {'target':'Id', 'source':'data', - 'path': 'FrobId'} + { + 'target': 'Id', + 'source': 'data', + 'path': 'FrobId' + } ] } } @@ -348,8 +355,11 @@ def test_action_beats_reference(self): 'resource': { 'type': 'Frob', 'identifiers': [ - {'target':'Id', 'source':'data', - 'path': 'FrobId'} + { + 'target': 'Id', + 'source': 'data', + 'path': 'FrobId' + } ] } } @@ -368,8 +378,11 @@ def test_reference_beats_collection(self): 'resource': { 'type': 'Frob', 'identifiers': [ - {'target':'Id', 'source':'data', - 'path': 'FrobId'} + { + 'target': 'Id', + 'source': 'data', + 'path': 'FrobId' + } ] } } diff --git a/tests/unit/resources/test_params.py b/tests/unit/resources/test_params.py index a78136bc70..84b43faed5 100644 --- a/tests/unit/resources/test_params.py +++ b/tests/unit/resources/test_params.py @@ -15,10 +15,12 @@ from boto3.exceptions import ResourceLoadException from boto3.resources.base import ResourceMeta, ServiceResource from boto3.resources.model import Request -from boto3.resources.params import create_request_parameters, \ - build_param_structure +from boto3.resources.params import ( + create_request_parameters, build_param_structure +) from tests import BaseTestCase, mock + class TestServiceActionParams(BaseTestCase): def test_service_action_params_identifier(self): request_model = Request({ @@ -104,7 +106,7 @@ def test_service_action_params_data_member_missing_no_load(self): parent.meta = ResourceMeta('test', data=None) with pytest.raises(ResourceLoadException): - params = create_request_parameters(parent, request_model) + create_request_parameters(parent, request_model) def test_service_action_params_constants(self): request_model = Request({ diff --git a/tests/unit/resources/test_response.py b/tests/unit/resources/test_response.py index 728a57264c..740f24175e 100644 --- a/tests/unit/resources/test_response.py +++ b/tests/unit/resources/test_response.py @@ -17,8 +17,9 @@ from boto3.resources.base import ResourceMeta, ServiceResource from boto3.resources.model import ResponseResource, Parameter from boto3.resources.factory import ResourceFactory -from boto3.resources.response import build_identifiers, build_empty_response,\ - RawHandler, ResourceHandler +from boto3.resources.response import ( + build_identifiers, build_empty_response, RawHandler, ResourceHandler +) class TestBuildIdentifiers(BaseTestCase): @@ -216,7 +217,6 @@ def test_path_structure(self): response = self.get_response() assert response is None - def test_path_list(self): self.search_path = 'Container[1].Frob' @@ -235,7 +235,6 @@ def test_path_list(self): response = self.get_response() assert response is None - def test_path_invalid(self): self.search_path = 'Container.Invalid' diff --git a/tests/unit/s3/test_inject.py b/tests/unit/s3/test_inject.py index 6d44517697..f167832059 100644 --- a/tests/unit/s3/test_inject.py +++ b/tests/unit/s3/test_inject.py @@ -100,6 +100,7 @@ def test_bucket_load_encounters_other_exception(self): with pytest.raises(ClientError): inject.bucket_load(self.resource) + class TestBucketTransferMethods(unittest.TestCase): def setUp(self): diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py index c471f78053..1fd7877f40 100644 --- a/tests/unit/test_session.py +++ b/tests/unit/test_session.py @@ -13,11 +13,11 @@ import pytest from botocore import loaders -from botocore.exceptions import DataNotFoundError, UnknownServiceError +from botocore.exceptions import UnknownServiceError from botocore.client import Config from boto3 import __version__ -from boto3.exceptions import NoVersionFound, ResourceNotExistsError +from boto3.exceptions import ResourceNotExistsError from boto3.session import Session from tests import mock, BaseTestCase @@ -116,7 +116,7 @@ def test_profile_default(self): def test_available_profiles(self): bc_session = mock.Mock() - bc_session.available_profiles.return_value = ['foo','bar'] + bc_session.available_profiles.return_value = ['foo', 'bar'] session = Session(botocore_session=bc_session) profiles = session.available_profiles assert len(profiles.return_value) == 2 @@ -245,7 +245,7 @@ def test_create_resource_with_args(self): config=mock.ANY) client_config = session.client.call_args[1]['config'] assert client_config.user_agent_extra == 'Resource' - assert client_config.signature_version == None + assert client_config.signature_version is None def test_create_resource_with_config(self): mock_bc_session = mock.Mock() From 471384e2cd738a746c549db7d1934b01058806ab Mon Sep 17 00:00:00 2001 From: aws-sdk-python-automation Date: Fri, 8 Oct 2021 18:20:23 +0000 Subject: [PATCH 3/4] Add changelog entries from botocore --- .changes/next-release/api-change-ec2-21089.json | 5 +++++ .changes/next-release/api-change-lexv2models-31838.json | 5 +++++ .changes/next-release/api-change-lexv2runtime-63419.json | 5 +++++ .changes/next-release/api-change-mediaconvert-10044.json | 5 +++++ .changes/next-release/api-change-secretsmanager-28876.json | 5 +++++ .changes/next-release/api-change-securityhub-56560.json | 5 +++++ 6 files changed, 30 insertions(+) create mode 100644 .changes/next-release/api-change-ec2-21089.json create mode 100644 .changes/next-release/api-change-lexv2models-31838.json create mode 100644 .changes/next-release/api-change-lexv2runtime-63419.json create mode 100644 .changes/next-release/api-change-mediaconvert-10044.json create mode 100644 .changes/next-release/api-change-secretsmanager-28876.json create mode 100644 .changes/next-release/api-change-securityhub-56560.json diff --git a/.changes/next-release/api-change-ec2-21089.json b/.changes/next-release/api-change-ec2-21089.json new file mode 100644 index 0000000000..0f0a30e41d --- /dev/null +++ b/.changes/next-release/api-change-ec2-21089.json @@ -0,0 +1,5 @@ +{ + "type": "api-change", + "category": "``ec2``", + "description": "[``botocore``] This release removes a requirement for filters on SearchLocalGatewayRoutes operations." +} diff --git a/.changes/next-release/api-change-lexv2models-31838.json b/.changes/next-release/api-change-lexv2models-31838.json new file mode 100644 index 0000000000..7d14fc27a2 --- /dev/null +++ b/.changes/next-release/api-change-lexv2models-31838.json @@ -0,0 +1,5 @@ +{ + "type": "api-change", + "category": "``lexv2-models``", + "description": "[``botocore``] Update lexv2-models client to latest version" +} diff --git a/.changes/next-release/api-change-lexv2runtime-63419.json b/.changes/next-release/api-change-lexv2runtime-63419.json new file mode 100644 index 0000000000..08503823e3 --- /dev/null +++ b/.changes/next-release/api-change-lexv2runtime-63419.json @@ -0,0 +1,5 @@ +{ + "type": "api-change", + "category": "``lexv2-runtime``", + "description": "[``botocore``] Update lexv2-runtime client to latest version" +} diff --git a/.changes/next-release/api-change-mediaconvert-10044.json b/.changes/next-release/api-change-mediaconvert-10044.json new file mode 100644 index 0000000000..1d470bb7fa --- /dev/null +++ b/.changes/next-release/api-change-mediaconvert-10044.json @@ -0,0 +1,5 @@ +{ + "type": "api-change", + "category": "``mediaconvert``", + "description": "[``botocore``] AWS Elemental MediaConvert has added the ability to set account policies which control access restrictions for HTTP, HTTPS, and S3 content sources." +} diff --git a/.changes/next-release/api-change-secretsmanager-28876.json b/.changes/next-release/api-change-secretsmanager-28876.json new file mode 100644 index 0000000000..faec915958 --- /dev/null +++ b/.changes/next-release/api-change-secretsmanager-28876.json @@ -0,0 +1,5 @@ +{ + "type": "api-change", + "category": "``secretsmanager``", + "description": "[``botocore``] Documentation updates for Secrets Manager" +} diff --git a/.changes/next-release/api-change-securityhub-56560.json b/.changes/next-release/api-change-securityhub-56560.json new file mode 100644 index 0000000000..652edc9bfb --- /dev/null +++ b/.changes/next-release/api-change-securityhub-56560.json @@ -0,0 +1,5 @@ +{ + "type": "api-change", + "category": "``securityhub``", + "description": "[``botocore``] Added new resource details objects to ASFF, including resources for WAF rate-based rules, EC2 VPC endpoints, ECR repositories, EKS clusters, X-Ray encryption, and OpenSearch domains. Added additional details for CloudFront distributions, CodeBuild projects, ELB V2 load balancers, and S3 buckets." +} From 1ad7bb310e03e971d0dc9e4eb15591f650e65ee1 Mon Sep 17 00:00:00 2001 From: aws-sdk-python-automation Date: Fri, 8 Oct 2021 18:20:31 +0000 Subject: [PATCH 4/4] Bumping version to 1.18.58 --- .changes/1.18.58.json | 32 +++++++++++++++++++ .../next-release/api-change-ec2-21089.json | 5 --- .../api-change-lexv2models-31838.json | 5 --- .../api-change-lexv2runtime-63419.json | 5 --- .../api-change-mediaconvert-10044.json | 5 --- .../api-change-secretsmanager-28876.json | 5 --- .../api-change-securityhub-56560.json | 5 --- CHANGELOG.rst | 12 +++++++ boto3/__init__.py | 2 +- setup.cfg | 2 +- setup.py | 2 +- 11 files changed, 47 insertions(+), 33 deletions(-) create mode 100644 .changes/1.18.58.json delete mode 100644 .changes/next-release/api-change-ec2-21089.json delete mode 100644 .changes/next-release/api-change-lexv2models-31838.json delete mode 100644 .changes/next-release/api-change-lexv2runtime-63419.json delete mode 100644 .changes/next-release/api-change-mediaconvert-10044.json delete mode 100644 .changes/next-release/api-change-secretsmanager-28876.json delete mode 100644 .changes/next-release/api-change-securityhub-56560.json diff --git a/.changes/1.18.58.json b/.changes/1.18.58.json new file mode 100644 index 0000000000..6ffdfda39e --- /dev/null +++ b/.changes/1.18.58.json @@ -0,0 +1,32 @@ +[ + { + "category": "``lexv2-runtime``", + "description": "[``botocore``] Update lexv2-runtime client to latest version", + "type": "api-change" + }, + { + "category": "``lexv2-models``", + "description": "[``botocore``] Update lexv2-models client to latest version", + "type": "api-change" + }, + { + "category": "``secretsmanager``", + "description": "[``botocore``] Documentation updates for Secrets Manager", + "type": "api-change" + }, + { + "category": "``securityhub``", + "description": "[``botocore``] Added new resource details objects to ASFF, including resources for WAF rate-based rules, EC2 VPC endpoints, ECR repositories, EKS clusters, X-Ray encryption, and OpenSearch domains. Added additional details for CloudFront distributions, CodeBuild projects, ELB V2 load balancers, and S3 buckets.", + "type": "api-change" + }, + { + "category": "``mediaconvert``", + "description": "[``botocore``] AWS Elemental MediaConvert has added the ability to set account policies which control access restrictions for HTTP, HTTPS, and S3 content sources.", + "type": "api-change" + }, + { + "category": "``ec2``", + "description": "[``botocore``] This release removes a requirement for filters on SearchLocalGatewayRoutes operations.", + "type": "api-change" + } +] \ No newline at end of file diff --git a/.changes/next-release/api-change-ec2-21089.json b/.changes/next-release/api-change-ec2-21089.json deleted file mode 100644 index 0f0a30e41d..0000000000 --- a/.changes/next-release/api-change-ec2-21089.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "type": "api-change", - "category": "``ec2``", - "description": "[``botocore``] This release removes a requirement for filters on SearchLocalGatewayRoutes operations." -} diff --git a/.changes/next-release/api-change-lexv2models-31838.json b/.changes/next-release/api-change-lexv2models-31838.json deleted file mode 100644 index 7d14fc27a2..0000000000 --- a/.changes/next-release/api-change-lexv2models-31838.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "type": "api-change", - "category": "``lexv2-models``", - "description": "[``botocore``] Update lexv2-models client to latest version" -} diff --git a/.changes/next-release/api-change-lexv2runtime-63419.json b/.changes/next-release/api-change-lexv2runtime-63419.json deleted file mode 100644 index 08503823e3..0000000000 --- a/.changes/next-release/api-change-lexv2runtime-63419.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "type": "api-change", - "category": "``lexv2-runtime``", - "description": "[``botocore``] Update lexv2-runtime client to latest version" -} diff --git a/.changes/next-release/api-change-mediaconvert-10044.json b/.changes/next-release/api-change-mediaconvert-10044.json deleted file mode 100644 index 1d470bb7fa..0000000000 --- a/.changes/next-release/api-change-mediaconvert-10044.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "type": "api-change", - "category": "``mediaconvert``", - "description": "[``botocore``] AWS Elemental MediaConvert has added the ability to set account policies which control access restrictions for HTTP, HTTPS, and S3 content sources." -} diff --git a/.changes/next-release/api-change-secretsmanager-28876.json b/.changes/next-release/api-change-secretsmanager-28876.json deleted file mode 100644 index faec915958..0000000000 --- a/.changes/next-release/api-change-secretsmanager-28876.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "type": "api-change", - "category": "``secretsmanager``", - "description": "[``botocore``] Documentation updates for Secrets Manager" -} diff --git a/.changes/next-release/api-change-securityhub-56560.json b/.changes/next-release/api-change-securityhub-56560.json deleted file mode 100644 index 652edc9bfb..0000000000 --- a/.changes/next-release/api-change-securityhub-56560.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "type": "api-change", - "category": "``securityhub``", - "description": "[``botocore``] Added new resource details objects to ASFF, including resources for WAF rate-based rules, EC2 VPC endpoints, ECR repositories, EKS clusters, X-Ray encryption, and OpenSearch domains. Added additional details for CloudFront distributions, CodeBuild projects, ELB V2 load balancers, and S3 buckets." -} diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 5de2efd7f0..6c2e852579 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,17 @@ CHANGELOG ========= +1.18.58 +======= + +* api-change:``lexv2-runtime``: [``botocore``] Update lexv2-runtime client to latest version +* api-change:``lexv2-models``: [``botocore``] Update lexv2-models client to latest version +* api-change:``secretsmanager``: [``botocore``] Documentation updates for Secrets Manager +* api-change:``securityhub``: [``botocore``] Added new resource details objects to ASFF, including resources for WAF rate-based rules, EC2 VPC endpoints, ECR repositories, EKS clusters, X-Ray encryption, and OpenSearch domains. Added additional details for CloudFront distributions, CodeBuild projects, ELB V2 load balancers, and S3 buckets. +* api-change:``mediaconvert``: [``botocore``] AWS Elemental MediaConvert has added the ability to set account policies which control access restrictions for HTTP, HTTPS, and S3 content sources. +* api-change:``ec2``: [``botocore``] This release removes a requirement for filters on SearchLocalGatewayRoutes operations. + + 1.18.57 ======= @@ -8249,3 +8260,4 @@ CHANGELOG * feature:Resources: Supports S3, EC2, SQS, SNS, and IAM resources * feature:Clients: Supports low-level clients for most services + diff --git a/boto3/__init__.py b/boto3/__init__.py index 422cfab7d9..851e01722a 100644 --- a/boto3/__init__.py +++ b/boto3/__init__.py @@ -18,7 +18,7 @@ __author__ = 'Amazon Web Services' -__version__ = '1.18.57' +__version__ = '1.18.58' # The default Boto3 session; autoloaded when needed. diff --git a/setup.cfg b/setup.cfg index 797e30b356..6068b77591 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,7 +3,7 @@ universal = 0 [metadata] requires_dist = - botocore>=1.21.57,<1.22.0 + botocore>=1.21.58,<1.22.0 jmespath>=0.7.1,<1.0.0 s3transfer>=0.5.0,<0.6.0 diff --git a/setup.py b/setup.py index 4a82492f41..3f4a81ea73 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ requires = [ - 'botocore>=1.21.57,<1.22.0', + 'botocore>=1.21.58,<1.22.0', 'jmespath>=0.7.1,<1.0.0', 's3transfer>=0.5.0,<0.6.0' ]