Skip to content

Commit

Permalink
add message attributes validation for SNS (#6938)
Browse files Browse the repository at this point in the history
  • Loading branch information
@bentsku
bentsku committed Sep 28, 2022
1 parent 60b19a0 commit 21b1687
Show file tree
Hide file tree
Showing 3 changed files with 282 additions and 4 deletions.
87 changes: 83 additions & 4 deletions localstack/services/sns/provider.py
View file
Expand Up @@ -4,9 +4,11 @@
import datetime
import json
import logging
import re
import time
import traceback
import uuid
from string import ascii_letters, digits
from typing import Dict, List

import botocore.exceptions
Expand Down Expand Up @@ -39,6 +41,7 @@
GetSubscriptionAttributesResponse,
GetTopicAttributesResponse,
InvalidParameterException,
InvalidParameterValueException,
LanguageCodeString,
ListEndpointsByPlatformApplicationResponse,
ListOriginationNumbersResult,
Expand Down Expand Up @@ -123,6 +126,9 @@

GCM_URL = "https://fcm.googleapis.com/fcm/send"

MSG_ATTR_NAME_REGEX = r"^(?!\.)(?!.*\.$)(?!.*\.\.)[a-zA-Z0-9_\-.]+$"
VALID_MSG_ATTR_NAME_CHARS = set(ascii_letters + digits + "." + "-" + "_")


class SNSBackend(RegionBackend):
# maps topic ARN to list of subscriptions
Expand Down Expand Up @@ -492,6 +498,10 @@ def publish_batch(
# TODO: implement SNS MessageDeduplicationId and ContentDeduplication checks

message_attributes = entry.get("MessageAttributes", {})
if message_attributes:
# if a message contains non-valid message attributes
# will fail for the first non-valid message encountered, and raise ParameterValueInvalid
validate_message_attributes(message_attributes)
try:
message_to_subscribers(
message_id,
Expand Down Expand Up @@ -697,6 +707,9 @@ def publish(
"Invalid parameter: MessageGroupId Reason: The request includes MessageGroupId parameter that is not valid for this topic type"
)

if message_attributes:
validate_message_attributes(message_attributes)

sns_backend = SNSBackend.get()
# No need to create a topic to send SMS or single push notifications with SNS
# but we can't mock a sending so we only return that it went well
Expand Down Expand Up @@ -1319,20 +1332,86 @@ def prepare_message_attributes(message_attributes: MessageAttributeMap):
# todo: Number type is not supported for Lambda subscriptions, passed as String
# do conversion here
for attr_name, attr in message_attributes.items():
if attr.get("StringValue", None):
val = attr["StringValue"]
else:
date_type = attr["DataType"]
if date_type == "Binary":
# binary payload in base64 encoded by AWS, UTF-8 for JSON
# https://docs.aws.amazon.com/sns/latest/api/API_MessageAttributeValue.html
val = base64.b64encode(attr["BinaryValue"]).decode()
else:
val = attr.get("StringValue")

attributes[attr_name] = {
"Type": attr["DataType"],
"Type": date_type,
"Value": val,
}
return attributes


def validate_message_attributes(message_attributes: MessageAttributeMap) -> None:
"""
Validate the message attributes, and raises an exception if those do not follow AWS validation
See: https://docs.aws.amazon.com/sns/latest/dg/sns-message-attributes.html
Regex from: https://stackoverflow.com/questions/40718851/regex-that-does-not-allow-consecutive-dots
:param message_attributes: the message attributes map for the message
:raises: InvalidParameterValueException
:return: None
"""
for attr_name, attr in message_attributes.items():
if len(attr_name) > 256:
raise InvalidParameterValueException(
"Length of message attribute name must be less than 256 bytes."
)
validate_message_attribute_name(attr_name)
# `DataType` is a required field for MessageAttributeValue
data_type = attr["DataType"]
if data_type not in ("String", "Number", "Binary", "String.Array"):
raise InvalidParameterValueException(
f"The message attribute '{attr_name}' has an invalid message attribute type, the set of supported type prefixes is Binary, Number, and String."
)
value_key_data_type = "Binary" if data_type == "Binary" else "String"
value_key = f"{value_key_data_type}Value"
if value_key not in attr:
raise InvalidParameterValueException(
f"The message attribute '{attr_name}' with type '{data_type}' must use field '{value_key_data_type}'."
)
elif not attr[value_key]:
raise InvalidParameterValueException(
f"The message attribute '{attr_name}' must contain non-empty message attribute value for message attribute type '{data_type}'.",
)


def validate_message_attribute_name(name: str) -> None:
"""
Validate the message attribute name with the specification of AWS.
The message attribute name can contain the following characters: A-Z, a-z, 0-9, underscore(_), hyphen(-), and period (.). The name must not start or end with a period, and it should not have successive periods.
:param name: message attribute name
:raises InvalidParameterValueException: if the name does not conform to the spec
"""
if not re.match(MSG_ATTR_NAME_REGEX, name):
# find the proper exception
if name[0] == ".":
raise InvalidParameterValueException(
"Invalid message attribute name starting with character '.' was found."
)
elif name[-1] == ".":
raise InvalidParameterValueException(
"Invalid message attribute name ending with character '.' was found."
)

for idx, char in enumerate(name):
if char not in VALID_MSG_ATTR_NAME_CHARS:
# change prefix from 0x to #x, without capitalizing the x
hex_char = "#x" + hex(ord(char)).upper()[2:]
raise InvalidParameterValueException(
f"Invalid non-alphanumeric character '{hex_char}' was found in the message attribute name. Can only include alphanumeric characters, hyphens, underscores, or dots."
)
# even if we go negative index, it will be covered by starting/ending with dot
if char == "." and name[idx - 1] == ".":
raise InvalidParameterValueException(
"Message attribute name can not have successive '.' character."
)


def create_subscribe_url(external_url, topic_arn, subscription_token):
return f"{external_url}/?Action=ConfirmSubscription&TopicArn={topic_arn}&Token={subscription_token}"

Expand Down
62 changes: 62 additions & 0 deletions tests/integration/test_sns.py
View file
Expand Up @@ -2326,3 +2326,65 @@ def validate_content():
sleep_before = 10

retry(validate_content, retries=retries, sleep_before=sleep_before, sleep=sleep)

@pytest.mark.aws_validated
def test_empty_or_wrong_message_attributes(
self,
sns_client,
sns_create_sqs_subscription,
sns_create_topic,
sqs_create_queue,
snapshot,
):
topic_arn = sns_create_topic()["TopicArn"]
queue_url = sqs_create_queue()

sns_create_sqs_subscription(topic_arn=topic_arn, queue_url=queue_url)

wrong_message_attributes = {
"missing_string_attr": {"attr1": {"DataType": "String", "StringValue": ""}},
"missing_binary_attr": {"attr1": {"DataType": "Binary", "BinaryValue": b""}},
"str_attr_binary_value": {"attr1": {"DataType": "String", "BinaryValue": b"123"}},
"int_attr_binary_value": {"attr1": {"DataType": "Number", "BinaryValue": b"123"}},
"binary_attr_string_value": {"attr1": {"DataType": "Binary", "StringValue": "123"}},
"invalid_attr_string_value": {
"attr1": {"DataType": "InvalidType", "StringValue": "123"}
},
"too_long_name": {"a" * 257: {"DataType": "String", "StringValue": "123"}},
"invalid_name": {"a^*?": {"DataType": "String", "StringValue": "123"}},
"invalid_name_2": {".abc": {"DataType": "String", "StringValue": "123"}},
"invalid_name_3": {"abc.": {"DataType": "String", "StringValue": "123"}},
"invalid_name_4": {"a..bc": {"DataType": "String", "StringValue": "123"}},
}

for error_type, msg_attrs in wrong_message_attributes.items():
with pytest.raises(ClientError) as e:
sns_client.publish(
TopicArn=topic_arn,
Message="test message",
MessageAttributes=msg_attrs,
)

snapshot.match(error_type, e.value.response)

with pytest.raises(ClientError) as e:
sns_client.publish_batch(
TopicArn=topic_arn,
PublishBatchRequestEntries=[
{
"Id": "1",
"Message": "test-batch",
"MessageAttributes": wrong_message_attributes["missing_string_attr"],
},
{
"Id": "2",
"Message": "test-batch",
"MessageAttributes": wrong_message_attributes["str_attr_binary_value"],
},
{
"Id": "3",
"Message": "valid-batch",
},
],
)
snapshot.match("batch-exception", e.value.response)
137 changes: 137 additions & 0 deletions tests/integration/test_sns.snapshot.json
View file
Expand Up @@ -2031,5 +2031,142 @@
}
}
}
},
"tests/integration/test_sns.py::TestSNSProvider::test_empty_or_wrong_message_attributes": {
"recorded-date": "27-09-2022, 16:22:03",
"recorded-content": {
"missing_string_attr": {
"Error": {
"Code": "ParameterValueInvalid",
"Message": "The message attribute 'attr1' must contain non-empty message attribute value for message attribute type 'String'.",
"Type": "Sender"
},
"ResponseMetadata": {
"HTTPHeaders": {},
"HTTPStatusCode": 400
}
},
"missing_binary_attr": {
"Error": {
"Code": "ParameterValueInvalid",
"Message": "The message attribute 'attr1' must contain non-empty message attribute value for message attribute type 'Binary'.",
"Type": "Sender"
},
"ResponseMetadata": {
"HTTPHeaders": {},
"HTTPStatusCode": 400
}
},
"str_attr_binary_value": {
"Error": {
"Code": "ParameterValueInvalid",
"Message": "The message attribute 'attr1' with type 'String' must use field 'String'.",
"Type": "Sender"
},
"ResponseMetadata": {
"HTTPHeaders": {},
"HTTPStatusCode": 400
}
},
"int_attr_binary_value": {
"Error": {
"Code": "ParameterValueInvalid",
"Message": "The message attribute 'attr1' with type 'Number' must use field 'String'.",
"Type": "Sender"
},
"ResponseMetadata": {
"HTTPHeaders": {},
"HTTPStatusCode": 400
}
},
"binary_attr_string_value": {
"Error": {
"Code": "ParameterValueInvalid",
"Message": "The message attribute 'attr1' with type 'Binary' must use field 'Binary'.",
"Type": "Sender"
},
"ResponseMetadata": {
"HTTPHeaders": {},
"HTTPStatusCode": 400
}
},
"invalid_attr_string_value": {
"Error": {
"Code": "ParameterValueInvalid",
"Message": "The message attribute 'attr1' has an invalid message attribute type, the set of supported type prefixes is Binary, Number, and String.",
"Type": "Sender"
},
"ResponseMetadata": {
"HTTPHeaders": {},
"HTTPStatusCode": 400
}
},
"too_long_name": {
"Error": {
"Code": "ParameterValueInvalid",
"Message": "Length of message attribute name must be less than 256 bytes.",
"Type": "Sender"
},
"ResponseMetadata": {
"HTTPHeaders": {},
"HTTPStatusCode": 400
}
},
"invalid_name": {
"Error": {
"Code": "ParameterValueInvalid",
"Message": "Invalid non-alphanumeric character '#x5E' was found in the message attribute name. Can only include alphanumeric characters, hyphens, underscores, or dots.",
"Type": "Sender"
},
"ResponseMetadata": {
"HTTPHeaders": {},
"HTTPStatusCode": 400
}
},
"invalid_name_2": {
"Error": {
"Code": "ParameterValueInvalid",
"Message": "Invalid message attribute name starting with character '.' was found.",
"Type": "Sender"
},
"ResponseMetadata": {
"HTTPHeaders": {},
"HTTPStatusCode": 400
}
},
"invalid_name_3": {
"Error": {
"Code": "ParameterValueInvalid",
"Message": "Invalid message attribute name ending with character '.' was found.",
"Type": "Sender"
},
"ResponseMetadata": {
"HTTPHeaders": {},
"HTTPStatusCode": 400
}
},
"invalid_name_4": {
"Error": {
"Code": "ParameterValueInvalid",
"Message": "Message attribute name can not have successive '.' character.",
"Type": "Sender"
},
"ResponseMetadata": {
"HTTPHeaders": {},
"HTTPStatusCode": 400
}
},
"batch-exception": {
"Error": {
"Code": "ParameterValueInvalid",
"Message": "The message attribute 'attr1' must contain non-empty message attribute value for message attribute type 'String'.",
"Type": "Sender"
},
"ResponseMetadata": {
"HTTPHeaders": {},
"HTTPStatusCode": 400
}
}
}
}
}

0 comments on commit 21b1687

Please sign in to comment.