Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

RFC/Draft for configuration utility #3546

Draft
wants to merge 1 commit into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,16 @@ install:
else
PYTHON_DOCKER_TAG=${TRAVIS_PYTHON_VERSION}-stretch
fi
docker run --rm -t --name motoserver -e TEST_SERVER_MODE=true -e AWS_SECRET_ACCESS_KEY=server_secret -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 5000:5000 -v /var/run/docker.sock:/var/run/docker.sock python:${PYTHON_DOCKER_TAG} /moto/travis_moto_server.sh &
docker run --rm -t \
--name motoserver \
-e MOTO_ENVIRONMENT=development \
-e TEST_SERVER_MODE=true \
-e AWS_SECRET_ACCESS_KEY=server_secret \
-e AWS_ACCESS_KEY_ID=server_key \
-v `pwd`:/moto \
-v /var/run/docker.sock:/var/run/docker.sock \
-p 5000:5000 \
python:${PYTHON_DOCKER_TAG} /moto/travis_moto_server.sh &
fi
travis_retry pip install -r requirements-dev.txt
travis_retry pip install docker>=2.5.1
Expand Down
1 change: 1 addition & 0 deletions MANIFEST.in
Original file line number Diff line number Diff line change
Expand Up @@ -5,5 +5,6 @@ include moto/ec2/resources/instance_type_offerings/*/*.json
include moto/ec2/resources/amis.json
include moto/cognitoidp/resources/*.json
include moto/dynamodb2/parsing/reserved_keywords.txt
include moto/configuration.ini
recursive-include moto/templates *
recursive-include tests *
4 changes: 2 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,12 @@ format:
test-only:
rm -f .coverage
rm -rf cover
@pytest -sv --cov=moto --cov-report html ./tests/ $(TEST_EXCLUDE)
@MOTO_ENVIRONMENT=development pytest -sv --cov=moto --cov-report html ./tests/ $(TEST_EXCLUDE)

test: lint test-only

test_server:
@TEST_SERVER_MODE=true pytest -sv --cov=moto --cov-report html ./tests/
@TEST_SERVER_MODE=true @MOTO_ENVIRONMENT=development pytest -sv --cov=moto --cov-report html ./tests/

aws_managed_policies:
scripts/update_managed_policies.py
Expand Down
5 changes: 5 additions & 0 deletions moto/configuration.ini
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
[production]
DeduplicationTimeInSeconds = 300

[development]
DeduplicationTimeInSeconds = 1
28 changes: 16 additions & 12 deletions moto/sqs/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
unix_time_millis,
tags_from_cloudformation_tags_list,
)
from .utils import generate_receipt_handle
from .utils import generate_receipt_handle, read_config
from .exceptions import (
MessageAttributesInvalid,
MessageNotInflight,
Expand Down Expand Up @@ -63,8 +63,6 @@
# https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-message-metadata.html
ATTRIBUTE_NAME_PATTERN = re.compile("^([a-z]|[A-Z]|[0-9]|[_.\\-])+$")

DEDUPLICATION_TIME_IN_SECONDS = 300


class Message(BaseModel):
def __init__(self, message_id, body):
Expand Down Expand Up @@ -231,9 +229,10 @@ class Queue(CloudFormationModel):
"SendMessage",
)

def __init__(self, name, region, **kwargs):
def __init__(self, name, region, config, **kwargs):
self.name = name
self.region = region
self.config = config
self.tags = {}
self.permissions = {}

Expand Down Expand Up @@ -489,7 +488,7 @@ def add_message(self, message):
diff = message.sent_timestamp - m.sent_timestamp
# if a duplicate message is received within the deduplication time then it should
# not be added to the queue
if diff / 1000 < DEDUPLICATION_TIME_IN_SECONDS:
if diff / 1000 < int(self.config["DeduplicationTimeInSeconds"]):
return

self._messages.append(message)
Expand Down Expand Up @@ -561,16 +560,18 @@ def _filter_message_attributes(message, input_message_attributes):


class SQSBackend(BaseBackend):
def __init__(self, region_name):
def __init__(self, region_name, config):
self.region_name = region_name
self.config = config
self.queues = {}
super(SQSBackend, self).__init__()

def reset(self):
region_name = self.region_name
config = self.config
self._reset_model_refs()
self.__dict__ = {}
self.__init__(region_name)
self.__init__(region_name, config)

def create_queue(self, name, tags=None, **kwargs):
queue = self.queues.get(name)
Expand All @@ -580,7 +581,9 @@ def create_queue(self, name, tags=None, **kwargs):
except KeyError:
pass

new_queue = Queue(name, region=self.region_name, **kwargs)
new_queue = Queue(
name, region=self.region_name, config=self.config, **kwargs
)

queue_attributes = queue.attributes
new_queue_attributes = new_queue.attributes
Expand All @@ -595,7 +598,7 @@ def create_queue(self, name, tags=None, **kwargs):
kwargs.pop("region")
except KeyError:
pass
queue = Queue(name, region=self.region_name, **kwargs)
queue = Queue(name, region=self.region_name, config=self.config, **kwargs)
self.queues[name] = queue

if tags:
Expand Down Expand Up @@ -1008,9 +1011,10 @@ def list_queue_tags(self, queue_name):


sqs_backends = {}
config = read_config()
for region in Session().get_available_regions("sqs"):
sqs_backends[region] = SQSBackend(region)
sqs_backends[region] = SQSBackend(region, config)
for region in Session().get_available_regions("sqs", partition_name="aws-us-gov"):
sqs_backends[region] = SQSBackend(region)
sqs_backends[region] = SQSBackend(region, config)
for region in Session().get_available_regions("sqs", partition_name="aws-cn"):
sqs_backends[region] = SQSBackend(region)
sqs_backends[region] = SQSBackend(region, config)
29 changes: 24 additions & 5 deletions moto/sqs/utils.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,17 @@
from __future__ import unicode_literals
import os
import random
import string
import moto

from .exceptions import MessageAttributesInvalid
from moto.settings import TEST_SERVER_MODE

try:
import configparser
except ModuleNotFoundError:
# why are you using python 2.7?
import ConfigParser as configparser


def generate_receipt_handle():
Expand Down Expand Up @@ -49,11 +58,7 @@ def parse_message_attributes(querystring, base="", value_namespace="Value."):
)

data_type_parts = data_type[0].split(".")
if data_type_parts[0] not in [
"String",
"Binary",
"Number",
]:
if data_type_parts[0] not in ["String", "Binary", "Number"]:
raise MessageAttributesInvalid(
"The message attribute '{0}' has an invalid message attribute type, the set of supported type prefixes is Binary, Number, and String.".format(
name[0]
Expand Down Expand Up @@ -83,3 +88,17 @@ def parse_message_attributes(querystring, base="", value_namespace="Value."):
index += 1

return message_attributes


def read_config():
section = os.getenv("MOTO_ENVIRONMENT", "production")
parser = configparser.ConfigParser()
filename = "configuration.ini"
config_path = os.path.join(os.getcwd(), "moto", filename)

if TEST_SERVER_MODE:
# in server mode the config resides in the package
config_path = os.path.join(os.path.dirname(moto.__file__), filename)

parser.read(config_path)
return parser[section]
16 changes: 2 additions & 14 deletions tests/test_sqs/test_sqs.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,7 @@
from moto import mock_sqs, mock_sqs_deprecated, mock_lambda, mock_logs, settings
from unittest import SkipTest

if sys.version_info[0] < 3:
import mock
from unittest import SkipTest
else:
from unittest import SkipTest, mock

import pytest
from tests.helpers import requires_boto_gte
from tests.test_awslambda.test_lambda import get_test_zip_file1, get_role_name
Expand Down Expand Up @@ -54,8 +50,6 @@
}
"""

MOCK_DEDUPLICATION_TIME_IN_SECONDS = 5


@mock_sqs
def test_create_fifo_queue_fail():
Expand Down Expand Up @@ -2343,22 +2337,16 @@ def test_fifo_queue_deduplication_withoutid(msg_1, msg_2, expected_count):
messages.should.have.length_of(expected_count)


@mock.patch(
"moto.sqs.models.DEDUPLICATION_TIME_IN_SECONDS", MOCK_DEDUPLICATION_TIME_IN_SECONDS
)
@mock_sqs
def test_fifo_queue_send_duplicate_messages_after_deduplication_time_limit():
if settings.TEST_SERVER_MODE:
raise SkipTest("Cant manipulate time in server mode")

sqs = boto3.resource("sqs", region_name="us-east-1")
msg_queue = sqs.create_queue(
QueueName="test-queue-dlq.fifo",
Attributes={"FifoQueue": "true", "ContentBasedDeduplication": "true"},
)

msg_queue.send_message(MessageBody="first", MessageGroupId="1")
time.sleep(MOCK_DEDUPLICATION_TIME_IN_SECONDS + 5)
time.sleep(5) # in tests deduplication time is 1 second
msg_queue.send_message(MessageBody="first", MessageGroupId="2")
messages = msg_queue.receive_messages(MaxNumberOfMessages=2)
messages.should.have.length_of(2)
Expand Down