Merge pull request #2519 from yan12125/python38

Python 3.8 support

.travis.yml

@@ -7,6 +7,7 @@ python:
 - 2.7
 - 3.6
 - 3.7
+- 3.8
 env:
 - TEST_SERVER_MODE=false
 - TEST_SERVER_MODE=true
@@ -17,7 +18,14 @@ install:
   python setup.py sdist
 
   if [ "$TEST_SERVER_MODE" = "true" ]; then
-    docker run --rm -t --name motoserver -e TEST_SERVER_MODE=true -e AWS_SECRET_ACCESS_KEY=server_secret -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 5000:5000 -v /var/run/docker.sock:/var/run/docker.sock python:${TRAVIS_PYTHON_VERSION}-stretch /moto/travis_moto_server.sh &
+    if [ "$TRAVIS_PYTHON_VERSION" = "3.8" ]; then
+      # Python 3.8 does not provide Stretch images yet [1]
+      # [1] https://github.com/docker-library/python/issues/428
+      PYTHON_DOCKER_TAG=${TRAVIS_PYTHON_VERSION}-buster
+    else
+      PYTHON_DOCKER_TAG=${TRAVIS_PYTHON_VERSION}-stretch
+    fi
+    docker run --rm -t --name motoserver -e TEST_SERVER_MODE=true -e AWS_SECRET_ACCESS_KEY=server_secret -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 5000:5000 -v /var/run/docker.sock:/var/run/docker.sock python:${PYTHON_DOCKER_TAG} /moto/travis_moto_server.sh &
   fi
   travis_retry pip install boto==2.45.0
   travis_retry pip install boto3

moto/__init__.py

@@ -1,59 +1,59 @@
 from __future__ import unicode_literals
-import logging
+# import logging
 # logging.getLogger('boto').setLevel(logging.CRITICAL)
 
 __title__ = 'moto'
 __version__ = '1.3.14.dev'
 
-from .acm import mock_acm  # flake8: noqa
-from .apigateway import mock_apigateway, mock_apigateway_deprecated  # flake8: noqa
-from .athena import mock_athena # flake8: noqa
-from .autoscaling import mock_autoscaling, mock_autoscaling_deprecated  # flake8: noqa
-from .awslambda import mock_lambda, mock_lambda_deprecated  # flake8: noqa
-from .cloudformation import mock_cloudformation, mock_cloudformation_deprecated  # flake8: noqa
-from .cloudwatch import mock_cloudwatch, mock_cloudwatch_deprecated  # flake8: noqa
-from .cognitoidentity import mock_cognitoidentity, mock_cognitoidentity_deprecated  # flake8: noqa
-from .cognitoidp import mock_cognitoidp, mock_cognitoidp_deprecated  # flake8: noqa
-from .config import mock_config  # flake8: noqa
-from .datapipeline import mock_datapipeline, mock_datapipeline_deprecated  # flake8: noqa
-from .dynamodb import mock_dynamodb, mock_dynamodb_deprecated  # flake8: noqa
-from .dynamodb2 import mock_dynamodb2, mock_dynamodb2_deprecated  # flake8: noqa
-from .dynamodbstreams import mock_dynamodbstreams # flake8: noqa
-from .ec2 import mock_ec2, mock_ec2_deprecated  # flake8: noqa
-from .ecr import mock_ecr, mock_ecr_deprecated  # flake8: noqa
-from .ecs import mock_ecs, mock_ecs_deprecated  # flake8: noqa
-from .elb import mock_elb, mock_elb_deprecated  # flake8: noqa
-from .elbv2 import mock_elbv2  # flake8: noqa
-from .emr import mock_emr, mock_emr_deprecated  # flake8: noqa
-from .events import mock_events  # flake8: noqa
-from .glacier import mock_glacier, mock_glacier_deprecated  # flake8: noqa
-from .glue import mock_glue  # flake8: noqa
-from .iam import mock_iam, mock_iam_deprecated  # flake8: noqa
-from .kinesis import mock_kinesis, mock_kinesis_deprecated  # flake8: noqa
-from .kms import mock_kms, mock_kms_deprecated  # flake8: noqa
-from .organizations import mock_organizations  # flake8: noqa
-from .opsworks import mock_opsworks, mock_opsworks_deprecated  # flake8: noqa
-from .polly import mock_polly  # flake8: noqa
-from .rds import mock_rds, mock_rds_deprecated  # flake8: noqa
-from .rds2 import mock_rds2, mock_rds2_deprecated  # flake8: noqa
-from .redshift import mock_redshift, mock_redshift_deprecated  # flake8: noqa
-from .resourcegroups import mock_resourcegroups  # flake8: noqa
-from .s3 import mock_s3, mock_s3_deprecated  # flake8: noqa
-from .ses import mock_ses, mock_ses_deprecated  # flake8: noqa
-from .secretsmanager import mock_secretsmanager  # flake8: noqa
-from .sns import mock_sns, mock_sns_deprecated  # flake8: noqa
-from .sqs import mock_sqs, mock_sqs_deprecated  # flake8: noqa
-from .stepfunctions import mock_stepfunctions # flake8: noqa
-from .sts import mock_sts, mock_sts_deprecated  # flake8: noqa
-from .ssm import mock_ssm  # flake8: noqa
-from .route53 import mock_route53, mock_route53_deprecated  # flake8: noqa
-from .swf import mock_swf, mock_swf_deprecated  # flake8: noqa
-from .xray import mock_xray, mock_xray_client, XRaySegment  # flake8: noqa
-from .logs import mock_logs, mock_logs_deprecated # flake8: noqa
-from .batch import mock_batch  # flake8: noqa
-from .resourcegroupstaggingapi import mock_resourcegroupstaggingapi  # flake8: noqa
-from .iot import mock_iot  # flake8: noqa
-from .iotdata import mock_iotdata  # flake8: noqa
+from .acm import mock_acm  # noqa
+from .apigateway import mock_apigateway, mock_apigateway_deprecated  # noqa
+from .athena import mock_athena # noqa
+from .autoscaling import mock_autoscaling, mock_autoscaling_deprecated  # noqa
+from .awslambda import mock_lambda, mock_lambda_deprecated  # noqa
+from .cloudformation import mock_cloudformation, mock_cloudformation_deprecated  # noqa
+from .cloudwatch import mock_cloudwatch, mock_cloudwatch_deprecated  # noqa
+from .cognitoidentity import mock_cognitoidentity, mock_cognitoidentity_deprecated  # noqa
+from .cognitoidp import mock_cognitoidp, mock_cognitoidp_deprecated  # noqa
+from .config import mock_config  # noqa
+from .datapipeline import mock_datapipeline, mock_datapipeline_deprecated  # noqa
+from .dynamodb import mock_dynamodb, mock_dynamodb_deprecated  # noqa
+from .dynamodb2 import mock_dynamodb2, mock_dynamodb2_deprecated  # noqa
+from .dynamodbstreams import mock_dynamodbstreams # noqa
+from .ec2 import mock_ec2, mock_ec2_deprecated  # noqa
+from .ecr import mock_ecr, mock_ecr_deprecated  # noqa
+from .ecs import mock_ecs, mock_ecs_deprecated  # noqa
+from .elb import mock_elb, mock_elb_deprecated  # noqa
+from .elbv2 import mock_elbv2  # noqa
+from .emr import mock_emr, mock_emr_deprecated  # noqa
+from .events import mock_events  # noqa
+from .glacier import mock_glacier, mock_glacier_deprecated  # noqa
+from .glue import mock_glue  # noqa
+from .iam import mock_iam, mock_iam_deprecated  # noqa
+from .kinesis import mock_kinesis, mock_kinesis_deprecated  # noqa
+from .kms import mock_kms, mock_kms_deprecated  # noqa
+from .organizations import mock_organizations  # noqa
+from .opsworks import mock_opsworks, mock_opsworks_deprecated  # noqa
+from .polly import mock_polly  # noqa
+from .rds import mock_rds, mock_rds_deprecated  # noqa
+from .rds2 import mock_rds2, mock_rds2_deprecated  # noqa
+from .redshift import mock_redshift, mock_redshift_deprecated  # noqa
+from .resourcegroups import mock_resourcegroups  # noqa
+from .s3 import mock_s3, mock_s3_deprecated  # noqa
+from .ses import mock_ses, mock_ses_deprecated  # noqa
+from .secretsmanager import mock_secretsmanager  # noqa
+from .sns import mock_sns, mock_sns_deprecated  # noqa
+from .sqs import mock_sqs, mock_sqs_deprecated  # noqa
+from .stepfunctions import mock_stepfunctions # noqa
+from .sts import mock_sts, mock_sts_deprecated  # noqa
+from .ssm import mock_ssm  # noqa
+from .route53 import mock_route53, mock_route53_deprecated  # noqa
+from .swf import mock_swf, mock_swf_deprecated  # noqa
+from .xray import mock_xray, mock_xray_client, XRaySegment  # noqa
+from .logs import mock_logs, mock_logs_deprecated # noqa
+from .batch import mock_batch  # noqa
+from .resourcegroupstaggingapi import mock_resourcegroupstaggingapi  # noqa
+from .iot import mock_iot  # noqa
+from .iotdata import mock_iotdata  # noqa
 
 
 try:

moto/cloudwatch/responses.py

@@ -96,11 +96,11 @@ def get_metric_statistics(self):
         extended_statistics = self._get_param('ExtendedStatistics')
         dimensions = self._get_param('Dimensions')
         if unit or extended_statistics or dimensions:
-            raise NotImplemented()
+            raise NotImplementedError()
 
         # TODO: this should instead throw InvalidParameterCombination
         if not statistics:
-            raise NotImplemented("Must specify either Statistics or ExtendedStatistics")
+            raise NotImplementedError("Must specify either Statistics or ExtendedStatistics")
 
         datapoints = self.cloudwatch_backend.get_metric_statistics(namespace, metric_name, start_time, end_time, period, statistics)
         template = self.response_template(GET_METRIC_STATISTICS_TEMPLATE)

moto/compat.py

@@ -1,5 +1,5 @@
 try:
-    from collections import OrderedDict  # flake8: noqa
+    from collections import OrderedDict  # noqa
 except ImportError:
     # python 2.6 or earlier, use backport
-    from ordereddict import OrderedDict  # flake8: noqa
+    from ordereddict import OrderedDict  # noqa

moto/core/__init__.py

@@ -1,6 +1,6 @@
 from __future__ import unicode_literals
 
-from .models import BaseModel, BaseBackend, moto_api_backend  # flake8: noqa
+from .models import BaseModel, BaseBackend, moto_api_backend  # noqa
 from .responses import ActionAuthenticatorMixin
 
 moto_api_backends = {"global": moto_api_backend}

moto/dynamodb2/comparisons.py

@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 import re
-import six
-import re
 from collections import deque
 from collections import namedtuple
 
@@ -48,11 +46,11 @@ def get_expected(expected):
         path = AttributePath([key])
         if 'Exists' in cond:
             if cond['Exists']:
-               conditions.append(FuncAttrExists(path))
+                conditions.append(FuncAttrExists(path))
             else:
-               conditions.append(FuncAttrNotExists(path))
+                conditions.append(FuncAttrNotExists(path))
         elif 'Value' in cond:
-           conditions.append(OpEqual(path, AttributeValue(cond['Value'])))
+            conditions.append(OpEqual(path, AttributeValue(cond['Value'])))
         elif 'ComparisonOperator' in cond:
             operator_name = cond['ComparisonOperator']
             values = [
@@ -91,12 +89,12 @@ def __repr__(self):
 
 # TODO add tests for all of these
 
-EQ_FUNCTION = lambda item_value, test_value: item_value == test_value  # flake8: noqa
-NE_FUNCTION = lambda item_value, test_value: item_value != test_value  # flake8: noqa
-LE_FUNCTION = lambda item_value, test_value: item_value <= test_value  # flake8: noqa
-LT_FUNCTION = lambda item_value, test_value: item_value < test_value  # flake8: noqa
-GE_FUNCTION = lambda item_value, test_value: item_value >= test_value  # flake8: noqa
-GT_FUNCTION = lambda item_value, test_value: item_value > test_value  # flake8: noqa
+EQ_FUNCTION = lambda item_value, test_value: item_value == test_value  # noqa
+NE_FUNCTION = lambda item_value, test_value: item_value != test_value  # noqa
+LE_FUNCTION = lambda item_value, test_value: item_value <= test_value  # noqa
+LT_FUNCTION = lambda item_value, test_value: item_value < test_value  # noqa
+GE_FUNCTION = lambda item_value, test_value: item_value >= test_value  # noqa
+GT_FUNCTION = lambda item_value, test_value: item_value > test_value  # noqa
 
 COMPARISON_FUNCS = {
     'EQ': EQ_FUNCTION,
@@ -221,7 +219,6 @@ class Kind:
         # --------------
         LITERAL = 'LITERAL'
 
-
     class Nonterminal:
         """Enum defining nonterminals for productions."""
 
@@ -240,7 +237,6 @@ class Nonterminal:
         RIGHT_PAREN = 'RIGHT_PAREN'
         WHITESPACE = 'WHITESPACE'
 
-
     Node = namedtuple('Node', ['nonterminal', 'kind', 'text', 'value', 'children'])
 
     def _lex_condition_expression(self):
@@ -286,11 +282,10 @@ def _lex_one_node(self, remaining_expression):
             if match:
                 match_text = match.group()
                 break
-        else: # pragma: no cover
+        else:  # pragma: no cover
             raise ValueError("Cannot parse condition starting at: " +
-                                 remaining_expression)
+                             remaining_expression)
 
-        value = match_text
         node = self.Node(
             nonterminal=nonterminal,
             kind=self.Kind.LITERAL,
@@ -351,7 +346,6 @@ def _parse_path_element(self, name):
             'size',
         }
 
-
         if name.lower() in reserved:
             # e.g. AND
             nonterminal = reserved[name.lower()]
@@ -748,14 +742,13 @@ def _make_operand(self, node):
         elif node.kind == self.Kind.FUNCTION:
             # size()
             function_node = node.children[0]
-            arguments =  node.children[1:]
+            arguments = node.children[1:]
             function_name = function_node.value
             arguments = [self._make_operand(arg) for arg in arguments]
             return FUNC_CLASS[function_name](*arguments)
-        else: # pragma: no cover
+        else:  # pragma: no cover
             raise ValueError("Unknown operand: %r" % node)
 
-
     def _make_op_condition(self, node):
         if node.kind == self.Kind.OR:
             lhs, rhs = node.children
@@ -796,7 +789,7 @@ def _make_op_condition(self, node):
             return COMPARATOR_CLASS[comparator.value](
                 self._make_operand(lhs),
                 self._make_operand(rhs))
-        else: # pragma: no cover
+        else:  # pragma: no cover
             raise ValueError("Unknown expression node kind %r" % node.kind)
 
     def _assert(self, condition, message, nodes):

moto/ec2/models.py

@@ -1229,7 +1229,7 @@ def describe_images(self, ami_ids=(), filters=None, exec_users=None, owners=None
 
             images = [ami for ami in images if ami.id in ami_ids]
             if len(images) == 0:
-                    raise InvalidAMIIdError(ami_ids)
+                raise InvalidAMIIdError(ami_ids)
         else:
             # Limit images by launch permissions
             if exec_users:
@@ -3720,8 +3720,8 @@ def __init__(self, ec2_backend, id, type,
         self.static_routes = None
 
     def get_filter_value(self, filter_name):
-            return super(VPNConnection, self).get_filter_value(
-                filter_name, 'DescribeVpnConnections')
+        return super(VPNConnection, self).get_filter_value(
+            filter_name, 'DescribeVpnConnections')
 
 
 class VPNConnectionBackend(object):
@@ -3950,8 +3950,8 @@ def __init__(self, ec2_backend, id, type):
         super(VpnGateway, self).__init__()
 
     def get_filter_value(self, filter_name):
-            return super(VpnGateway, self).get_filter_value(
-                filter_name, 'DescribeVpnGateways')
+        return super(VpnGateway, self).get_filter_value(
+            filter_name, 'DescribeVpnGateways')
 
 
 class VpnGatewayAttachment(object):
@@ -4015,8 +4015,8 @@ def __init__(self, ec2_backend, id, type, ip_address, bgp_asn):
         super(CustomerGateway, self).__init__()
 
     def get_filter_value(self, filter_name):
-            return super(CustomerGateway, self).get_filter_value(
-                filter_name, 'DescribeCustomerGateways')
+        return super(CustomerGateway, self).get_filter_value(
+            filter_name, 'DescribeCustomerGateways')
 
 
 class CustomerGatewayBackend(object):

moto/ecr/models.py

@@ -2,7 +2,6 @@
 
 import hashlib
 import re
-from copy import copy
 from datetime import datetime
 from random import random
 
@@ -27,11 +26,12 @@ def camelCase(self, key):
         return ''.join(words)
 
     def gen_response_object(self):
-        response_object = copy(self.__dict__)
-        for key, value in response_object.items():
+        response_object = dict()
+        for key, value in self.__dict__.items():
             if '_' in key:
                 response_object[self.camelCase(key)] = value
-                del response_object[key]
+            else:
+                response_object[key] = value
         return response_object
 
     @property

moto/ecs/models.py

@@ -158,8 +158,8 @@ def update_from_cloudformation_json(cls, original_resource, new_resource_name, c
         if (original_resource.family != family or
                 original_resource.container_definitions != container_definitions or
                 original_resource.volumes != volumes):
-                # currently TaskRoleArn isn't stored at TaskDefinition
-                # instances
+            # currently TaskRoleArn isn't stored at TaskDefinition
+            # instances
             ecs_backend = ecs_backends[region_name]
             ecs_backend.deregister_task_definition(original_resource.arn)
             return ecs_backend.register_task_definition(

moto/iotdata/models.py

@@ -163,7 +163,7 @@ def update_thing_shadow(self, thing_name, payload):
             raise InvalidRequestException('State contains an invalid node')
 
         if 'version' in payload and thing.thing_shadow.version != payload['version']:
-                raise ConflictException('Version conflict')
+            raise ConflictException('Version conflict')
         new_shadow = FakeShadow.create_from_previous_version(thing.thing_shadow, payload)
         thing.thing_shadow = new_shadow
         return thing.thing_shadow

moto/rds2/models.py

@@ -875,8 +875,8 @@ def stop_database(self, db_instance_identifier, db_snapshot_identifier=None):
         database = self.describe_databases(db_instance_identifier)[0]
         # todo: certain rds types not allowed to be stopped at this time.
         if database.is_replica or database.multi_az:
-                # todo: more db types not supported by stop/start instance api
-                raise InvalidDBClusterStateFaultError(db_instance_identifier)
+            # todo: more db types not supported by stop/start instance api
+            raise InvalidDBClusterStateFaultError(db_instance_identifier)
         if database.status != 'available':
             raise InvalidDBInstanceStateError(db_instance_identifier, 'stop')
         if db_snapshot_identifier:

moto/resourcegroupstaggingapi/models.py

@@ -244,7 +244,7 @@ def get_elbv2_tags(arn):
         def get_kms_tags(kms_key_id):
             result = []
             for tag in self.kms_backend.list_resource_tags(kms_key_id):
-                    result.append({'Key': tag['TagKey'], 'Value': tag['TagValue']})
+                result.append({'Key': tag['TagKey'], 'Value': tag['TagValue']})
             return result
 
         if not resource_type_filters or 'kms' in resource_type_filters:

moto/ssm/models.py

@@ -210,9 +210,9 @@ def get_invocation(self, instance_id, plugin_name):
                 'An error occurred (InvocationDoesNotExist) when calling the GetCommandInvocation operation')
 
         if plugin_name is not None and invocation['PluginName'] != plugin_name:
-                raise RESTError(
-                    'InvocationDoesNotExist',
-                    'An error occurred (InvocationDoesNotExist) when calling the GetCommandInvocation operation')
+            raise RESTError(
+                'InvocationDoesNotExist',
+                'An error occurred (InvocationDoesNotExist) when calling the GetCommandInvocation operation')
 
         return invocation