From 57ac73714db7204aac38959282d7cd0ec67cd7dd Mon Sep 17 00:00:00 2001 From: kentdr Date: Thu, 9 Mar 2023 15:57:51 -0500 Subject: [PATCH 01/27] Update TFM to net6.0 --- .../NServiceBus.AwsLambda.SQS.AcceptanceTests.csproj | 2 +- .../NServiceBus.AwsLambda.SQS.Tests.csproj | 2 +- src/NServiceBus.AwsLambda.SQS/LambdaLog.cs | 2 ++ src/NServiceBus.AwsLambda.SQS/MessageExtensions.cs | 2 +- src/NServiceBus.AwsLambda.SQS/NServiceBus.AwsLambda.SQS.csproj | 2 +- src/NServiceBus.AwsLambda.SQS/TransportMessage.cs | 2 +- 6 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/NServiceBus.AwsLambda.SQS.AcceptanceTests.csproj b/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/NServiceBus.AwsLambda.SQS.AcceptanceTests.csproj index 59d2520..f09806a 100644 --- a/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/NServiceBus.AwsLambda.SQS.AcceptanceTests.csproj +++ b/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/NServiceBus.AwsLambda.SQS.AcceptanceTests.csproj @@ -1,7 +1,7 @@  - netcoreapp3.1;net7.0 + net6.0 true NServiceBus.AwsLambda.Tests diff --git a/src/NServiceBus.AwsLambda.SQS.Tests/NServiceBus.AwsLambda.SQS.Tests.csproj b/src/NServiceBus.AwsLambda.SQS.Tests/NServiceBus.AwsLambda.SQS.Tests.csproj index c5f8abc..c62f985 100644 --- a/src/NServiceBus.AwsLambda.SQS.Tests/NServiceBus.AwsLambda.SQS.Tests.csproj +++ b/src/NServiceBus.AwsLambda.SQS.Tests/NServiceBus.AwsLambda.SQS.Tests.csproj @@ -1,7 +1,7 @@  - netcoreapp3.1;net7.0 + net6.0 true NServiceBus.AwsLambda.Tests diff --git a/src/NServiceBus.AwsLambda.SQS/LambdaLog.cs b/src/NServiceBus.AwsLambda.SQS/LambdaLog.cs index 266bcea..85b3466 100644 --- a/src/NServiceBus.AwsLambda.SQS/LambdaLog.cs +++ b/src/NServiceBus.AwsLambda.SQS/LambdaLog.cs @@ -4,6 +4,8 @@ namespace NServiceBus.AwsLambda.SQS using Amazon.Lambda.Core; using NServiceBus.Logging; + using LogLevel = Logging.LogLevel; + class LambdaLog : ILog { string name; diff --git a/src/NServiceBus.AwsLambda.SQS/MessageExtensions.cs b/src/NServiceBus.AwsLambda.SQS/MessageExtensions.cs index ca409f3..a17121b 100644 --- a/src/NServiceBus.AwsLambda.SQS/MessageExtensions.cs +++ b/src/NServiceBus.AwsLambda.SQS/MessageExtensions.cs @@ -31,7 +31,7 @@ static class MessageExtensions using (var memoryStream = new MemoryStream()) { - await s3GetResponse.ResponseStream.CopyToAsync(memoryStream).ConfigureAwait(false); + await s3GetResponse.ResponseStream.CopyToAsync(memoryStream, cancellationToken).ConfigureAwait(false); return memoryStream.ToArray(); } } diff --git a/src/NServiceBus.AwsLambda.SQS/NServiceBus.AwsLambda.SQS.csproj b/src/NServiceBus.AwsLambda.SQS/NServiceBus.AwsLambda.SQS.csproj index 67ff10c..308249a 100644 --- a/src/NServiceBus.AwsLambda.SQS/NServiceBus.AwsLambda.SQS.csproj +++ b/src/NServiceBus.AwsLambda.SQS/NServiceBus.AwsLambda.SQS.csproj @@ -1,7 +1,7 @@  - netstandard2.0 + net6.0 true ..\NServiceBus.snk diff --git a/src/NServiceBus.AwsLambda.SQS/TransportMessage.cs b/src/NServiceBus.AwsLambda.SQS/TransportMessage.cs index 69e3a7a..9e61324 100644 --- a/src/NServiceBus.AwsLambda.SQS/TransportMessage.cs +++ b/src/NServiceBus.AwsLambda.SQS/TransportMessage.cs @@ -54,7 +54,7 @@ public string TimeToBeReceived public Address? ReplyToAddress { - get => Headers.ContainsKey(NServiceBus.Headers.ReplyToAddress) ? new Address { Queue = Headers[NServiceBus.Headers.ReplyToAddress] } : (Address?)null; + get => Headers.ContainsKey(NServiceBus.Headers.ReplyToAddress) ? new Address { Queue = Headers[NServiceBus.Headers.ReplyToAddress] } : null; set { if (!string.IsNullOrWhiteSpace(value?.Queue)) From 3c5e6f0b596bdf418c14f14086d71afbddaf32af Mon Sep 17 00:00:00 2001 From: kentdr Date: Fri, 10 Mar 2023 16:36:14 -0500 Subject: [PATCH 02/27] Update AmazonSQS transport --- .../AwsLambdaSQSEndpointTestBase.cs | 19 +++-- ...SEvent_with_large_payloads_is_processed.cs | 11 +-- .../When_a_handler_sends_a_message.cs | 14 ++-- .../When_a_message_handler_always_throws.cs | 7 +- .../When_the_SQS_trigger_is_activated.cs | 7 +- .../APIApprovals.Approve.approved.txt | 6 +- .../AwsLambdaEndpoint.cs | 51 +++++------ .../AwsLambdaSQSEndpointConfiguration.cs | 84 ++++++++++--------- .../NServiceBus.AwsLambda.SQS.csproj | 2 +- .../TransportMessage.cs | 5 +- .../ManualPipelineInvocationInfrastructure.cs | 15 ---- .../TransportWrapper/NoOpQueueCreator.cs | 13 --- .../TransportWrapper/PipelineInvoker.cs | 47 +++++++---- .../TransportWrapper/ServerlessTransport.cs | 47 ++++++++--- .../ServerlessTransportInfrastructure.cs | 51 +++-------- 15 files changed, 191 insertions(+), 188 deletions(-) delete mode 100644 src/NServiceBus.AwsLambda.SQS/TransportWrapper/ManualPipelineInvocationInfrastructure.cs delete mode 100644 src/NServiceBus.AwsLambda.SQS/TransportWrapper/NoOpQueueCreator.cs diff --git a/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/AwsLambdaSQSEndpointTestBase.cs b/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/AwsLambdaSQSEndpointTestBase.cs index 038eb8b..6997fbc 100644 --- a/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/AwsLambdaSQSEndpointTestBase.cs +++ b/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/AwsLambdaSQSEndpointTestBase.cs @@ -9,6 +9,7 @@ using Amazon.Runtime; using Amazon.S3; using Amazon.S3.Model; + using Amazon.SimpleNotificationService; using Amazon.SQS; using Amazon.SQS.Model; using NUnit.Framework; @@ -90,11 +91,13 @@ protected async Task GenerateAndReceiveSQSEvent(int count) where T { var endpointConfiguration = new EndpointConfiguration($"{QueueNamePrefix}sender"); endpointConfiguration.SendOnly(); - endpointConfiguration.UsePersistence(); - var transport = endpointConfiguration.UseTransport(); - transport.ClientFactory(CreateSQSClient); - var s3 = transport.S3(BucketName, KeyPrefix); - s3.ClientFactory(CreateS3Client); + + var transport = new SqsTransport(CreateSQSClient(), null) + { + S3 = new S3Settings(BucketName, KeyPrefix, CreateS3Client()) + }; + + endpointConfiguration.UseTransport(transport); var endpointInstance = await Endpoint.Start(endpointConfiguration) .ConfigureAwait(false); @@ -133,6 +136,12 @@ public static IAmazonSQS CreateSQSClient() return new AmazonSQSClient(credentials); } + public static IAmazonSimpleNotificationService CreateSNSClient() + { + var credentials = new EnvironmentVariablesAWSCredentials(); + return new AmazonSimpleNotificationServiceClient(credentials); + } + public static IAmazonS3 CreateS3Client() { var credentials = new EnvironmentVariablesAWSCredentials(); diff --git a/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/When_a_SQSEvent_with_large_payloads_is_processed.cs b/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/When_a_SQSEvent_with_large_payloads_is_processed.cs index 47f8172..70a662f 100644 --- a/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/When_a_SQSEvent_with_large_payloads_is_processed.cs +++ b/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/When_a_SQSEvent_with_large_payloads_is_processed.cs @@ -2,6 +2,7 @@ { using System.Threading; using System.Threading.Tasks; + using Microsoft.Extensions.DependencyInjection; using NUnit.Framework; class When_a_SQSEvent_with_large_payloads_is_processed : AwsLambdaSQSEndpointTestBase @@ -15,16 +16,16 @@ public async Task The_handlers_should_be_invoked_and_process_successfully() var endpoint = new AwsLambdaSQSEndpoint(ctx => { - var configuration = new AwsLambdaSQSEndpointConfiguration(QueueName); + var configuration = new AwsLambdaSQSEndpointConfiguration(QueueName, CreateSQSClient(), CreateSNSClient()); var transport = configuration.Transport; - transport.ClientFactory(CreateSQSClient); - var s3 = transport.S3(BucketName, KeyPrefix); - s3.ClientFactory(CreateS3Client); + + transport.S3 = new S3Settings(BucketName, KeyPrefix, CreateS3Client()); + var advanced = configuration.AdvancedConfiguration; advanced.SendFailedMessagesTo(ErrorQueueName); - advanced.RegisterComponents(c => c.RegisterSingleton(typeof(TestContext), context)); + advanced.RegisterComponents(c => c.AddSingleton(typeof(TestContext), context)); return configuration; }); diff --git a/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/When_a_handler_sends_a_message.cs b/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/When_a_handler_sends_a_message.cs index 829814a..806e574 100644 --- a/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/When_a_handler_sends_a_message.cs +++ b/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/When_a_handler_sends_a_message.cs @@ -1,6 +1,7 @@ namespace NServiceBus.AwsLambda.Tests { using System.Threading.Tasks; + using Microsoft.Extensions.DependencyInjection; using NUnit.Framework; class When_a_handler_sends_a_message : AwsLambdaSQSEndpointTestBase @@ -16,26 +17,23 @@ public async Task The_message_should_be_received() RegisterQueueNameToCleanup(destinationEndpointName); var destinationConfiguration = new EndpointConfiguration(destinationEndpointName); - destinationConfiguration.UsePersistence(); + var destinationTransport = destinationConfiguration.UseTransport(); destinationTransport.ClientFactory(CreateSQSClient); destinationConfiguration.SendFailedMessagesTo(ErrorQueueName); destinationConfiguration.EnableInstallers(); - destinationConfiguration.RegisterComponents(c => c.RegisterSingleton(typeof(TestContext), context)); + destinationConfiguration.RegisterComponents(c => c.AddSingleton(typeof(TestContext), context)); var destinationEndpoint = await Endpoint.Start(destinationConfiguration); var endpoint = new AwsLambdaSQSEndpoint(ctx => { - var configuration = new AwsLambdaSQSEndpointConfiguration(QueueName); - var transport = configuration.Transport; - transport.ClientFactory(CreateSQSClient); + var configuration = new AwsLambdaSQSEndpointConfiguration(QueueName, CreateSQSClient(), CreateSNSClient()); - var routing = transport.Routing(); - routing.RouteToEndpoint(typeof(SentMessage), destinationEndpointName); + configuration.RoutingSettings.RouteToEndpoint(typeof(SentMessage), destinationEndpointName); var advanced = configuration.AdvancedConfiguration; advanced.SendFailedMessagesTo(ErrorQueueName); - advanced.RegisterComponents(c => c.RegisterSingleton(typeof(TestContext), context)); + advanced.RegisterComponents(c => c.AddSingleton(typeof(TestContext), context)); return configuration; }); diff --git a/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/When_a_message_handler_always_throws.cs b/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/When_a_message_handler_always_throws.cs index a4e4dca..387dafe 100644 --- a/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/When_a_message_handler_always_throws.cs +++ b/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/When_a_message_handler_always_throws.cs @@ -3,6 +3,7 @@ using System; using System.Threading; using System.Threading.Tasks; + using Microsoft.Extensions.DependencyInjection; using NUnit.Framework; class When_a_message_handler_always_throws : AwsLambdaSQSEndpointTestBase @@ -16,13 +17,11 @@ public async Task The_messages_should_forward_to_error_queue_by_default() var endpoint = new AwsLambdaSQSEndpoint(ctx => { - var configuration = new AwsLambdaSQSEndpointConfiguration(QueueName); - var transport = configuration.Transport; - transport.ClientFactory(CreateSQSClient); + var configuration = new AwsLambdaSQSEndpointConfiguration(QueueName, CreateSQSClient(), CreateSNSClient()); var advanced = configuration.AdvancedConfiguration; advanced.SendFailedMessagesTo(ErrorQueueName); - advanced.RegisterComponents(c => c.RegisterSingleton(typeof(TestContext), context)); + advanced.RegisterComponents(c => c.AddSingleton(typeof(TestContext), context)); return configuration; }); diff --git a/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/When_the_SQS_trigger_is_activated.cs b/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/When_the_SQS_trigger_is_activated.cs index 065995b..94e4eab 100644 --- a/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/When_the_SQS_trigger_is_activated.cs +++ b/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/When_the_SQS_trigger_is_activated.cs @@ -2,6 +2,7 @@ { using System.Threading; using System.Threading.Tasks; + using Microsoft.Extensions.DependencyInjection; using NUnit.Framework; class When_a_SQSEvent_is_processed : AwsLambdaSQSEndpointTestBase @@ -15,13 +16,11 @@ public async Task The_handlers_should_be_invoked_and_process_successfully() var endpoint = new AwsLambdaSQSEndpoint(ctx => { - var configuration = new AwsLambdaSQSEndpointConfiguration(QueueName); - var transport = configuration.Transport; - transport.ClientFactory(CreateSQSClient); + var configuration = new AwsLambdaSQSEndpointConfiguration(QueueName, CreateSQSClient(), CreateSNSClient()); var advanced = configuration.AdvancedConfiguration; advanced.SendFailedMessagesTo(ErrorQueueName); - advanced.RegisterComponents(c => c.RegisterSingleton(typeof(TestContext), context)); + advanced.RegisterComponents(c => c.AddSingleton(typeof(TestContext), context)); return configuration; }); diff --git a/src/NServiceBus.AwsLambda.SQS.Tests/ApprovalFiles/APIApprovals.Approve.approved.txt b/src/NServiceBus.AwsLambda.SQS.Tests/ApprovalFiles/APIApprovals.Approve.approved.txt index 87c093a..1adef98 100644 --- a/src/NServiceBus.AwsLambda.SQS.Tests/ApprovalFiles/APIApprovals.Approve.approved.txt +++ b/src/NServiceBus.AwsLambda.SQS.Tests/ApprovalFiles/APIApprovals.Approve.approved.txt @@ -20,13 +20,13 @@ namespace NServiceBus public class AwsLambdaSQSEndpointConfiguration { public AwsLambdaSQSEndpointConfiguration(string endpointName) { } + public AwsLambdaSQSEndpointConfiguration(string endpointName, Amazon.SQS.IAmazonSQS sqsClient, Amazon.SimpleNotificationService.IAmazonSimpleNotificationService snsClient) { } public NServiceBus.EndpointConfiguration AdvancedConfiguration { get; } - public NServiceBus.TransportExtensions Transport { get; } + public NServiceBus.RoutingSettings RoutingSettings { get; } + public NServiceBus.SqsTransport Transport { get; } public void DoNotSendMessagesToErrorQueue() { } public NServiceBus.Serialization.SerializationExtensions UseSerialization() where T : NServiceBus.Serialization.SerializationDefinition, new () { } - protected NServiceBus.TransportExtensions UseTransport() - where TTransport : NServiceBus.Transport.TransportDefinition, new () { } } public interface IAwsLambdaSQSEndpoint { diff --git a/src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs b/src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs index e924861..135c306 100644 --- a/src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs +++ b/src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs @@ -60,12 +60,14 @@ async Task InitializeEndpointIfNecessary(ILambdaContext executionContext, Cancel if (pipeline == null) { var configuration = configurationFactory(executionContext); + var serverlessTransport = configuration.MakeServerless(); + await Initialize(configuration).ConfigureAwait(false); LogManager.GetLogger("Previews").Info("NServiceBus.AwsLambda.SQS is a preview package. Preview packages are licensed separately from the rest of the Particular Software platform and have different support guarantees. You can view the license at https://particular.net/eula/previews and the support policy at https://docs.particular.net/previews/support-policy. Customer adoption drives whether NServiceBus.AwsLambda.SQS will be incorporated into the Particular Software platform. Let us know you are using it, if you haven't already, by emailing us at support@particular.net."); - endpoint = await Endpoint.Start(configuration.EndpointConfiguration).ConfigureAwait(false); + endpoint = await Endpoint.Start(configuration.EndpointConfiguration, token).ConfigureAwait(false); - pipeline = configuration.PipelineInvoker; + pipeline = serverlessTransport.PipelineInvoker; } } finally @@ -172,21 +174,21 @@ public async Task Unsubscribe(Type eventType, ILambdaContext lambdaContext) async Task Initialize(AwsLambdaSQSEndpointConfiguration configuration) { var settingsHolder = configuration.AdvancedConfiguration.GetSettings(); - var sqsClientFactory = settingsHolder.GetOrDefault>(SettingsKeys.SqsClientFactory) ?? (() => new AmazonSQSClient()); - sqsClient = sqsClientFactory(); + sqsClient = configuration.Transport.SqsClient; awsEndpointUrl = sqsClient.Config.DetermineServiceURL(); + queueUrl = await GetQueueUrl(settingsHolder.EndpointName()).ConfigureAwait(false); errorQueueUrl = await GetQueueUrl(settingsHolder.ErrorQueueAddress()).ConfigureAwait(false); - s3BucketForLargeMessages = settingsHolder.GetOrDefault(SettingsKeys.S3BucketForLargeMessages); + + + s3BucketForLargeMessages = configuration.Transport.S3?.BucketName; if (string.IsNullOrWhiteSpace(s3BucketForLargeMessages)) { return; } - - var s3ClientFactory = settingsHolder.GetOrDefault>(SettingsKeys.S3ClientFactory) ?? (() => new AmazonS3Client()); - s3Client = s3ClientFactory(); + s3Client = configuration.Transport.S3?.S3Client; } async Task GetQueueUrl(string queueName) @@ -285,7 +287,7 @@ static bool IsMessageExpired(SQSEvent.SQSMessage receivedMessage, Dictionary headers, string nativeMessageId, byte[] body, ILambdaContext lambdaContext, CancellationToken token) + async Task ProcessMessageWithInMemoryRetries(Dictionary headers, string nativeMessageId, ReadOnlyMemory body, ILambdaContext lambdaContext, CancellationToken token) { var immediateProcessingAttempts = 0; var messageProcessedOk = false; @@ -295,20 +297,18 @@ async Task ProcessMessageWithInMemoryRetries(Dictionary headers, { try { - using (var messageContextCancellationTokenSource = new CancellationTokenSource()) - { - var messageContext = new MessageContext( - nativeMessageId, - new Dictionary(headers), - body, - transportTransaction, - messageContextCancellationTokenSource, - new ContextBag()); + var messageContext = new MessageContext( + nativeMessageId, + new Dictionary(headers), + body, + transportTransaction, + queueUrl, + new ContextBag()); - await Process(messageContext, lambdaContext).ConfigureAwait(false); + await Process(messageContext, lambdaContext, token).ConfigureAwait(false); + + messageProcessedOk = !token.IsCancellationRequested; - messageProcessedOk = !messageContextCancellationTokenSource.IsCancellationRequested; - } } catch (Exception ex) when (!(ex is OperationCanceledException && token.IsCancellationRequested)) @@ -324,7 +324,10 @@ async Task ProcessMessageWithInMemoryRetries(Dictionary headers, nativeMessageId, body, transportTransaction, - immediateProcessingAttempts); + immediateProcessingAttempts, + queueUrl, + new ContextBag() + ); errorHandlerResult = await ProcessFailedMessage(errorContext, lambdaContext).ConfigureAwait(false); } @@ -339,9 +342,9 @@ async Task ProcessMessageWithInMemoryRetries(Dictionary headers, } } - async Task Process(MessageContext messageContext, ILambdaContext executionContext) + async Task Process(MessageContext messageContext, ILambdaContext executionContext, CancellationToken cancellationToken) { - await InitializeEndpointIfNecessary(executionContext, messageContext.ReceiveCancellationTokenSource.Token).ConfigureAwait(false); + await InitializeEndpointIfNecessary(executionContext, cancellationToken).ConfigureAwait(false); await pipeline.PushMessage(messageContext).ConfigureAwait(false); } diff --git a/src/NServiceBus.AwsLambda.SQS/AwsLambdaSQSEndpointConfiguration.cs b/src/NServiceBus.AwsLambda.SQS/AwsLambdaSQSEndpointConfiguration.cs index 010636e..08cfafd 100644 --- a/src/NServiceBus.AwsLambda.SQS/AwsLambdaSQSEndpointConfiguration.cs +++ b/src/NServiceBus.AwsLambda.SQS/AwsLambdaSQSEndpointConfiguration.cs @@ -2,12 +2,12 @@ { using System; using System.Threading.Tasks; + using Amazon.SimpleNotificationService; + using Amazon.SQS; using AwsLambda.SQS; - using AwsLambda.SQS.TransportWrapper; - using Configuration.AdvancedExtensibility; + using NServiceBus.AwsLambda.SQS.TransportWrapper; using NServiceBus.Logging; using Serialization; - using Transport; /// /// Represents a serverless NServiceBus endpoint running with an AmazonSQS SQS trigger. @@ -21,27 +21,47 @@ public class AwsLambdaSQSEndpointConfiguration /// /// The endpoint name to be used. public AwsLambdaSQSEndpointConfiguration(string endpointName) + : this(endpointName, null, null) { - EndpointConfiguration = new EndpointConfiguration(endpointName); - EndpointConfiguration.UsePersistence(); + } + + /// + /// Creates a serverless NServiceBus endpoint running with an AmazonSQS SQS trigger. + /// + public AwsLambdaSQSEndpointConfiguration(string endpointName, IAmazonSQS sqsClient, IAmazonSimpleNotificationService snsClient) + { + EndpointConfiguration = new EndpointConfiguration(endpointName); LogManager.Use(); - //make sure a call to "onError" will move the message to the error queue. - EndpointConfiguration.Recoverability().Delayed(c => c.NumberOfRetries(0)); - // send failed messages to the error queue recoverabilityPolicy.SendFailedMessagesToErrorQueue = true; - EndpointConfiguration.Recoverability().CustomPolicy(recoverabilityPolicy.Invoke); - Transport = UseTransport(); + //make sure a call to "onError" will move the message to the error queue. + EndpointConfiguration.Recoverability() + .Delayed(c => c.NumberOfRetries(0)) + .CustomPolicy(recoverabilityPolicy.Invoke); + + if (sqsClient is null && snsClient is null) + { + // If both sqsClient/snsClient are null, use default constructor which will instantiate a default client instance. + Transport = new SqsTransport(); + } + else + { + // If either sqsClient or snsClient is null the transport will throw. + Transport = new SqsTransport(sqsClient, snsClient); + } + + RoutingSettings = EndpointConfiguration.UseTransport(Transport); // by default do not write custom diagnostics to file because lambda is readonly - AdvancedConfiguration.CustomDiagnosticsWriter(diagnostics => Task.CompletedTask); + AdvancedConfiguration.CustomDiagnosticsWriter((diagnostics, token) => Task.CompletedTask); TrySpecifyDefaultLicense(); } + void TrySpecifyDefaultLicense() { var licenseText = Environment.GetEnvironmentVariable("NSERVICEBUS_LICENSE"); @@ -52,30 +72,30 @@ void TrySpecifyDefaultLicense() } /// - /// Amazon SQS transport + /// Amazon SQS transport routing settings /// - public TransportExtensions Transport { get; } + public RoutingSettings RoutingSettings { get; } + + /// + /// Amazon SQS Transport + /// + public SqsTransport Transport { get; } internal EndpointConfiguration EndpointConfiguration { get; } - internal PipelineInvoker PipelineInvoker { get; private set; } + + internal ServerlessTransport MakeServerless() + { + var serverlessTransport = new ServerlessTransport(Transport); + EndpointConfiguration.UseTransport(serverlessTransport); + + return serverlessTransport; + } /// /// Gives access to the underlying endpoint configuration for advanced configuration options. /// public EndpointConfiguration AdvancedConfiguration => EndpointConfiguration; - /// - /// Define a transport to be used when sending and publishing messages. - /// - protected TransportExtensions UseTransport() - where TTransport : TransportDefinition, new() - { - var serverlessTransport = EndpointConfiguration.UseTransport>(); - //TODO improve - PipelineInvoker = PipelineAccess(serverlessTransport); - return BaseTransportConfiguration(serverlessTransport); - } - /// /// Define the serializer to be used. /// @@ -91,17 +111,5 @@ public void DoNotSendMessagesToErrorQueue() { recoverabilityPolicy.SendFailedMessagesToErrorQueue = false; } - - static PipelineInvoker PipelineAccess( - TransportExtensions> transportConfiguration) where TBaseTransport : TransportDefinition, new() - { - return transportConfiguration.GetSettings().GetOrCreate(); - } - - static TransportExtensions BaseTransportConfiguration( - TransportExtensions> transportConfiguration) where TBaseTransport : TransportDefinition, new() - { - return new TransportExtensions(transportConfiguration.GetSettings()); - } } } \ No newline at end of file diff --git a/src/NServiceBus.AwsLambda.SQS/NServiceBus.AwsLambda.SQS.csproj b/src/NServiceBus.AwsLambda.SQS/NServiceBus.AwsLambda.SQS.csproj index 308249a..a21d958 100644 --- a/src/NServiceBus.AwsLambda.SQS/NServiceBus.AwsLambda.SQS.csproj +++ b/src/NServiceBus.AwsLambda.SQS/NServiceBus.AwsLambda.SQS.csproj @@ -9,7 +9,7 @@ - + diff --git a/src/NServiceBus.AwsLambda.SQS/TransportMessage.cs b/src/NServiceBus.AwsLambda.SQS/TransportMessage.cs index 9e61324..0202617 100644 --- a/src/NServiceBus.AwsLambda.SQS/TransportMessage.cs +++ b/src/NServiceBus.AwsLambda.SQS/TransportMessage.cs @@ -3,7 +3,6 @@ using System; using System.Collections.Generic; using System.Linq; - using DeliveryConstraints; using Performance.TimeToBeReceived; using Transport; @@ -14,7 +13,7 @@ public TransportMessage() { } - public TransportMessage(OutgoingMessage outgoingMessage, List deliveryConstraints) + public TransportMessage(OutgoingMessage outgoingMessage, List deliveryConstraints) { Headers = outgoingMessage.Headers; @@ -31,7 +30,7 @@ public TransportMessage(OutgoingMessage outgoingMessage, List Headers { get; set; } diff --git a/src/NServiceBus.AwsLambda.SQS/TransportWrapper/ManualPipelineInvocationInfrastructure.cs b/src/NServiceBus.AwsLambda.SQS/TransportWrapper/ManualPipelineInvocationInfrastructure.cs deleted file mode 100644 index 005e615..0000000 --- a/src/NServiceBus.AwsLambda.SQS/TransportWrapper/ManualPipelineInvocationInfrastructure.cs +++ /dev/null @@ -1,15 +0,0 @@ -namespace NServiceBus.AwsLambda.SQS.TransportWrapper -{ - using System.Threading.Tasks; - using Transport; - - class ManualPipelineInvocationInfrastructure : TransportReceiveInfrastructure - { - public ManualPipelineInvocationInfrastructure(PipelineInvoker pipelineInvoker) : - base(() => pipelineInvoker, - () => new NoOpQueueCreator(), - () => Task.FromResult(StartupCheckResult.Success)) - { - } - } -} \ No newline at end of file diff --git a/src/NServiceBus.AwsLambda.SQS/TransportWrapper/NoOpQueueCreator.cs b/src/NServiceBus.AwsLambda.SQS/TransportWrapper/NoOpQueueCreator.cs deleted file mode 100644 index 4237de5..0000000 --- a/src/NServiceBus.AwsLambda.SQS/TransportWrapper/NoOpQueueCreator.cs +++ /dev/null @@ -1,13 +0,0 @@ -namespace NServiceBus.AwsLambda.SQS.TransportWrapper -{ - using System.Threading.Tasks; - using Transport; - - class NoOpQueueCreator : ICreateQueues - { - public Task CreateQueueIfNecessary(QueueBindings queueBindings, string identity) - { - return Task.CompletedTask; - } - } -} \ No newline at end of file diff --git a/src/NServiceBus.AwsLambda.SQS/TransportWrapper/PipelineInvoker.cs b/src/NServiceBus.AwsLambda.SQS/TransportWrapper/PipelineInvoker.cs index fae0da2..cffd651 100644 --- a/src/NServiceBus.AwsLambda.SQS/TransportWrapper/PipelineInvoker.cs +++ b/src/NServiceBus.AwsLambda.SQS/TransportWrapper/PipelineInvoker.cs @@ -1,30 +1,40 @@ namespace NServiceBus.AwsLambda.SQS.TransportWrapper { - using System; + using System.Threading; using System.Threading.Tasks; using Transport; - class PipelineInvoker : IPushMessages + class PipelineInvoker : IMessageReceiver { - Task IPushMessages.Init(Func onMessage, Func> onError, CriticalError criticalError, PushSettings settings) + public PipelineInvoker(IMessageReceiver baseTransportReceiver) { - if (this.onMessage == null) - { - // The core ReceiveComponent calls TransportInfrastructure.MessagePumpFactory() multiple times - // the first invocation is for the main pipeline, ignore all other pipelines as we don't want to manually invoke them. - this.onMessage = onMessage; + this.baseTransportReceiver = baseTransportReceiver; + } - this.onError = onError; - } + public ISubscriptionManager Subscriptions => baseTransportReceiver.Subscriptions; - return Task.CompletedTask; + + public string Id => baseTransportReceiver.Id; + + public string ReceiveAddress => baseTransportReceiver.ReceiveAddress; + + Task IMessageReceiver.Initialize(PushRuntimeSettings limitations, OnMessage onMessage, OnError onError, CancellationToken cancellationToken) + { + this.onMessage = onMessage; + this.onError = onError; + + return baseTransportReceiver?.Initialize(limitations, + (_, __) => Task.CompletedTask, + (_, __) => Task.FromResult(ErrorHandleResult.Handled), + cancellationToken) ?? Task.CompletedTask; } - void IPushMessages.Start(PushRuntimeSettings limitations) + Task IMessageReceiver.StartReceive(CancellationToken cancellationToken) { + return Task.CompletedTask; } - Task IPushMessages.Stop() + Task IMessageReceiver.StopReceive(CancellationToken cancellationToken) { return Task.CompletedTask; } @@ -34,12 +44,19 @@ public Task PushFailedMessage(ErrorContext errorContext) return onError(errorContext); } + Task IMessageReceiver.ChangeConcurrency(PushRuntimeSettings limitations, CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + public Task PushMessage(MessageContext messageContext) { return onMessage.Invoke(messageContext); } - Func onMessage; - Func> onError; + OnMessage onMessage; + OnError onError; + + readonly IMessageReceiver baseTransportReceiver; } } \ No newline at end of file diff --git a/src/NServiceBus.AwsLambda.SQS/TransportWrapper/ServerlessTransport.cs b/src/NServiceBus.AwsLambda.SQS/TransportWrapper/ServerlessTransport.cs index 7e41134..7535171 100644 --- a/src/NServiceBus.AwsLambda.SQS/TransportWrapper/ServerlessTransport.cs +++ b/src/NServiceBus.AwsLambda.SQS/TransportWrapper/ServerlessTransport.cs @@ -1,26 +1,53 @@ namespace NServiceBus.AwsLambda.SQS.TransportWrapper { - using Settings; + using System.Collections.Generic; + using System.Threading; + using System.Threading.Tasks; + using NServiceBus; using Transport; class ServerlessTransport : TransportDefinition - where TBaseTransport : TransportDefinition, new() + where TBaseTransport : TransportDefinition { - public ServerlessTransport() + // HINT: This constant is defined in NServiceBus but is not exposed + const string MainReceiverId = "Main"; + + public ServerlessTransport(TBaseTransport baseTransport) + : base(baseTransport.TransportTransactionMode, baseTransport.SupportsDelayedDelivery, baseTransport.SupportsPublishSubscribe, baseTransport.SupportsTTBR) { - baseTransport = new TBaseTransport(); + BaseTransport = baseTransport; } - public override string ExampleConnectionStringForErrorMessage { get; } = string.Empty; + public TBaseTransport BaseTransport { get; } - public override bool RequiresConnectionString => baseTransport.RequiresConnectionString; + public PipelineInvoker PipelineInvoker { get; private set; } - public override TransportInfrastructure Initialize(SettingsHolder settings, string connectionString) + public override async Task Initialize(HostSettings hostSettings, ReceiveSettings[] receivers, string[] sendingAddresses, CancellationToken cancellationToken = default) { - var baseTransportInfrastructure = baseTransport.Initialize(settings, connectionString); - return new ServerlessTransportInfrastructure(baseTransportInfrastructure, settings); + // hostSettings. + var baseTransportInfrastructure = await BaseTransport.Initialize(hostSettings, receivers, sendingAddresses, cancellationToken).ConfigureAwait(false); + var serverlessTransportInfrastructure = new ServerlessTransportInfrastructure(baseTransportInfrastructure); + PipelineInvoker = (PipelineInvoker)serverlessTransportInfrastructure.Receivers[MainReceiverId]; + + return serverlessTransportInfrastructure; + } - readonly TBaseTransport baseTransport; +#pragma warning disable CS0672 // Member overrides obsolete member +#pragma warning disable CS0618 // Type or memeber is obsolete + + public override string ToTransportAddress(QueueAddress address) => BaseTransport.ToTransportAddress(address); + +#pragma warning restore CS0618 // Type or memeber is obsolete +#pragma warning restore CS0672 // Member overrides obsolete member + + + public override IReadOnlyCollection GetSupportedTransactionModes() => supportedTransactionModes; + + readonly TransportTransactionMode[] supportedTransactionModes = + { + TransportTransactionMode.None, + TransportTransactionMode.ReceiveOnly + }; } } \ No newline at end of file diff --git a/src/NServiceBus.AwsLambda.SQS/TransportWrapper/ServerlessTransportInfrastructure.cs b/src/NServiceBus.AwsLambda.SQS/TransportWrapper/ServerlessTransportInfrastructure.cs index e565970..b67e3b8 100644 --- a/src/NServiceBus.AwsLambda.SQS/TransportWrapper/ServerlessTransportInfrastructure.cs +++ b/src/NServiceBus.AwsLambda.SQS/TransportWrapper/ServerlessTransportInfrastructure.cs @@ -1,56 +1,27 @@ namespace NServiceBus.AwsLambda.SQS.TransportWrapper { - using System; - using System.Collections.Generic; - using Routing; - using Settings; + using System.Linq; + using System.Threading; + using System.Threading.Tasks; using Transport; - class ServerlessTransportInfrastructure : TransportInfrastructure + class ServerlessTransportInfrastructure : TransportInfrastructure where TBaseInfrastructure : TransportDefinition { - public ServerlessTransportInfrastructure(TransportInfrastructure baseTransportInfrastructure, - SettingsHolder settings) + public ServerlessTransportInfrastructure(TransportInfrastructure baseTransportInfrastructure) { this.baseTransportInfrastructure = baseTransportInfrastructure; - this.settings = settings; + Dispatcher = baseTransportInfrastructure.Dispatcher; + Receivers = baseTransportInfrastructure.Receivers.ToDictionary(r => r.Key, r => (IMessageReceiver)new PipelineInvoker(r.Value)); } - public override IEnumerable DeliveryConstraints => - baseTransportInfrastructure.DeliveryConstraints; - + //TODO: Is this still necessary? //support ReceiveOnly so that we can use immediate retries - public override TransportTransactionMode TransactionMode { get; } = TransportTransactionMode.ReceiveOnly; - - public override OutboundRoutingPolicy OutboundRoutingPolicy => - baseTransportInfrastructure.OutboundRoutingPolicy; - - public override TransportReceiveInfrastructure ConfigureReceiveInfrastructure() - { - var pipelineInvoker = settings.GetOrCreate(); - return new ManualPipelineInvocationInfrastructure(pipelineInvoker); - } + public TransportTransactionMode TransactionMode { get; } = TransportTransactionMode.ReceiveOnly; - public override TransportSendInfrastructure ConfigureSendInfrastructure() - { - return baseTransportInfrastructure.ConfigureSendInfrastructure(); - } + public override Task Shutdown(CancellationToken cancellationToken = default) => baseTransportInfrastructure.Shutdown(cancellationToken); - public override TransportSubscriptionInfrastructure ConfigureSubscriptionInfrastructure() - { - return baseTransportInfrastructure.ConfigureSubscriptionInfrastructure(); - } - - public override EndpointInstance BindToLocalEndpoint(EndpointInstance instance) - { - return baseTransportInfrastructure.BindToLocalEndpoint(instance); - } - - public override string ToTransportAddress(LogicalAddress logicalAddress) - { - return baseTransportInfrastructure.ToTransportAddress(logicalAddress); - } + public override string ToTransportAddress(QueueAddress address) => baseTransportInfrastructure.ToTransportAddress(address); readonly TransportInfrastructure baseTransportInfrastructure; - readonly SettingsHolder settings; } } \ No newline at end of file From da5e08d3e0a383c5b175f4ad8106f26866881497 Mon Sep 17 00:00:00 2001 From: kentdr Date: Tue, 14 Mar 2023 08:47:20 -0400 Subject: [PATCH 03/27] Singularize the TFM tag for projects --- .../NServiceBus.AwsLambda.SQS.AcceptanceTests.csproj | 2 +- .../NServiceBus.AwsLambda.SQS.Tests.csproj | 2 +- src/NServiceBus.AwsLambda.SQS/NServiceBus.AwsLambda.SQS.csproj | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/NServiceBus.AwsLambda.SQS.AcceptanceTests.csproj b/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/NServiceBus.AwsLambda.SQS.AcceptanceTests.csproj index f09806a..8d17fea 100644 --- a/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/NServiceBus.AwsLambda.SQS.AcceptanceTests.csproj +++ b/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/NServiceBus.AwsLambda.SQS.AcceptanceTests.csproj @@ -1,7 +1,7 @@  - net6.0 + net6.0 true NServiceBus.AwsLambda.Tests diff --git a/src/NServiceBus.AwsLambda.SQS.Tests/NServiceBus.AwsLambda.SQS.Tests.csproj b/src/NServiceBus.AwsLambda.SQS.Tests/NServiceBus.AwsLambda.SQS.Tests.csproj index c62f985..2850980 100644 --- a/src/NServiceBus.AwsLambda.SQS.Tests/NServiceBus.AwsLambda.SQS.Tests.csproj +++ b/src/NServiceBus.AwsLambda.SQS.Tests/NServiceBus.AwsLambda.SQS.Tests.csproj @@ -1,7 +1,7 @@  - net6.0 + net6.0 true NServiceBus.AwsLambda.Tests diff --git a/src/NServiceBus.AwsLambda.SQS/NServiceBus.AwsLambda.SQS.csproj b/src/NServiceBus.AwsLambda.SQS/NServiceBus.AwsLambda.SQS.csproj index a21d958..259fbf5 100644 --- a/src/NServiceBus.AwsLambda.SQS/NServiceBus.AwsLambda.SQS.csproj +++ b/src/NServiceBus.AwsLambda.SQS/NServiceBus.AwsLambda.SQS.csproj @@ -1,7 +1,7 @@  - net6.0 + net6.0 true ..\NServiceBus.snk From ba06dad6263c6d4d6139f1d73ec82cf960ec4d39 Mon Sep 17 00:00:00 2001 From: kentdr Date: Tue, 14 Mar 2023 08:47:41 -0400 Subject: [PATCH 04/27] Set the SNS client in tests instead of null --- .../AwsLambdaSQSEndpointTestBase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/AwsLambdaSQSEndpointTestBase.cs b/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/AwsLambdaSQSEndpointTestBase.cs index 6997fbc..ba7d2f9 100644 --- a/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/AwsLambdaSQSEndpointTestBase.cs +++ b/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/AwsLambdaSQSEndpointTestBase.cs @@ -92,7 +92,7 @@ protected async Task GenerateAndReceiveSQSEvent(int count) where T var endpointConfiguration = new EndpointConfiguration($"{QueueNamePrefix}sender"); endpointConfiguration.SendOnly(); - var transport = new SqsTransport(CreateSQSClient(), null) + var transport = new SqsTransport(CreateSQSClient(), CreateSNSClient()) { S3 = new S3Settings(BucketName, KeyPrefix, CreateS3Client()) }; From c589cf1e4823f1a45dcd8ae8278e2d83b3be27a9 Mon Sep 17 00:00:00 2001 From: kentdr Date: Fri, 24 Mar 2023 13:59:23 -0400 Subject: [PATCH 05/27] Update TFM to add net7.0 --- .../NServiceBus.AwsLambda.SQS.AcceptanceTests.csproj | 2 +- .../NServiceBus.AwsLambda.SQS.Tests.csproj | 2 +- src/NServiceBus.AwsLambda.SQS/NServiceBus.AwsLambda.SQS.csproj | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/NServiceBus.AwsLambda.SQS.AcceptanceTests.csproj b/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/NServiceBus.AwsLambda.SQS.AcceptanceTests.csproj index 8d17fea..972c368 100644 --- a/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/NServiceBus.AwsLambda.SQS.AcceptanceTests.csproj +++ b/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/NServiceBus.AwsLambda.SQS.AcceptanceTests.csproj @@ -1,7 +1,7 @@  - net6.0 + net6.0;net7.0 true NServiceBus.AwsLambda.Tests diff --git a/src/NServiceBus.AwsLambda.SQS.Tests/NServiceBus.AwsLambda.SQS.Tests.csproj b/src/NServiceBus.AwsLambda.SQS.Tests/NServiceBus.AwsLambda.SQS.Tests.csproj index 2850980..79087c2 100644 --- a/src/NServiceBus.AwsLambda.SQS.Tests/NServiceBus.AwsLambda.SQS.Tests.csproj +++ b/src/NServiceBus.AwsLambda.SQS.Tests/NServiceBus.AwsLambda.SQS.Tests.csproj @@ -1,7 +1,7 @@  - net6.0 + net6.0;net7.0 true NServiceBus.AwsLambda.Tests diff --git a/src/NServiceBus.AwsLambda.SQS/NServiceBus.AwsLambda.SQS.csproj b/src/NServiceBus.AwsLambda.SQS/NServiceBus.AwsLambda.SQS.csproj index 259fbf5..7671bc2 100644 --- a/src/NServiceBus.AwsLambda.SQS/NServiceBus.AwsLambda.SQS.csproj +++ b/src/NServiceBus.AwsLambda.SQS/NServiceBus.AwsLambda.SQS.csproj @@ -1,7 +1,7 @@  - net6.0 + net6.0;net7.0 true ..\NServiceBus.snk From b525699b22be2860ca33fe615475de8fd68b2a48 Mon Sep 17 00:00:00 2001 From: kentdr Date: Fri, 24 Mar 2023 14:09:08 -0400 Subject: [PATCH 06/27] Remove dotnet 3.1 from CI, add 6.0.x --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fdacfb1..854538b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,7 +39,7 @@ jobs: with: dotnet-version: | 7.0.x - 3.1.x + 6.0.x - name: Build run: dotnet build src --configuration Release - name: Upload packages From 4f5ea5251813ad7be7c88ceece8db1cdee05c648 Mon Sep 17 00:00:00 2001 From: kentdr Date: Fri, 24 Mar 2023 14:16:35 -0400 Subject: [PATCH 07/27] Change to discard operators --- .../AwsLambdaSQSEndpointConfiguration.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/NServiceBus.AwsLambda.SQS/AwsLambdaSQSEndpointConfiguration.cs b/src/NServiceBus.AwsLambda.SQS/AwsLambdaSQSEndpointConfiguration.cs index 08cfafd..70a07f9 100644 --- a/src/NServiceBus.AwsLambda.SQS/AwsLambdaSQSEndpointConfiguration.cs +++ b/src/NServiceBus.AwsLambda.SQS/AwsLambdaSQSEndpointConfiguration.cs @@ -56,7 +56,7 @@ public AwsLambdaSQSEndpointConfiguration(string endpointName, IAmazonSQS sqsClie RoutingSettings = EndpointConfiguration.UseTransport(Transport); // by default do not write custom diagnostics to file because lambda is readonly - AdvancedConfiguration.CustomDiagnosticsWriter((diagnostics, token) => Task.CompletedTask); + AdvancedConfiguration.CustomDiagnosticsWriter((_, _) => Task.CompletedTask); TrySpecifyDefaultLicense(); } From cce0497826402ecbbb2e54c933a25ad23d143263 Mon Sep 17 00:00:00 2001 From: kentdr Date: Fri, 24 Mar 2023 14:28:17 -0400 Subject: [PATCH 08/27] Fix whitespace --- src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs b/src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs index 135c306..739adb1 100644 --- a/src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs +++ b/src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs @@ -181,8 +181,6 @@ async Task Initialize(AwsLambdaSQSEndpointConfiguration configuration) queueUrl = await GetQueueUrl(settingsHolder.EndpointName()).ConfigureAwait(false); errorQueueUrl = await GetQueueUrl(settingsHolder.ErrorQueueAddress()).ConfigureAwait(false); - - s3BucketForLargeMessages = configuration.Transport.S3?.BucketName; if (string.IsNullOrWhiteSpace(s3BucketForLargeMessages)) { From ff3770494d9106bfaf98388809480093d4337bc1 Mon Sep 17 00:00:00 2001 From: kentdr Date: Fri, 24 Mar 2023 14:28:39 -0400 Subject: [PATCH 09/27] Change DispatchProperties paramter on TransportMessage --- src/NServiceBus.AwsLambda.SQS/TransportMessage.cs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/NServiceBus.AwsLambda.SQS/TransportMessage.cs b/src/NServiceBus.AwsLambda.SQS/TransportMessage.cs index 0202617..2152713 100644 --- a/src/NServiceBus.AwsLambda.SQS/TransportMessage.cs +++ b/src/NServiceBus.AwsLambda.SQS/TransportMessage.cs @@ -2,8 +2,6 @@ { using System; using System.Collections.Generic; - using System.Linq; - using Performance.TimeToBeReceived; using Transport; class TransportMessage @@ -13,7 +11,7 @@ public TransportMessage() { } - public TransportMessage(OutgoingMessage outgoingMessage, List deliveryConstraints) + public TransportMessage(OutgoingMessage outgoingMessage, DispatchProperties deliveryConstraints) { Headers = outgoingMessage.Headers; @@ -24,10 +22,9 @@ public TransportMessage(OutgoingMessage outgoingMessage, List().SingleOrDefault(); - if (discardConstraint != null) + if (deliveryConstraints.DiscardIfNotReceivedBefore != null) { - TimeToBeReceived = discardConstraint.MaxTime.ToString(); + TimeToBeReceived = deliveryConstraints.DiscardIfNotReceivedBefore.MaxTime.ToString(); } Body = !outgoingMessage.Body.IsEmpty ? Convert.ToBase64String(outgoingMessage.Body.Span) : "empty message"; From 066d2ee5aa173c87326635651422743b425de054 Mon Sep 17 00:00:00 2001 From: kentdr Date: Mon, 27 Mar 2023 11:39:37 -0400 Subject: [PATCH 10/27] Remove generics from ServerlessTransport and ServerlessTransportInfrastructure --- .../AwsLambdaSQSEndpointConfiguration.cs | 4 ++-- .../TransportWrapper/ServerlessTransport.cs | 9 ++++----- .../ServerlessTransportInfrastructure.cs | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/NServiceBus.AwsLambda.SQS/AwsLambdaSQSEndpointConfiguration.cs b/src/NServiceBus.AwsLambda.SQS/AwsLambdaSQSEndpointConfiguration.cs index 70a07f9..bea0e21 100644 --- a/src/NServiceBus.AwsLambda.SQS/AwsLambdaSQSEndpointConfiguration.cs +++ b/src/NServiceBus.AwsLambda.SQS/AwsLambdaSQSEndpointConfiguration.cs @@ -83,9 +83,9 @@ void TrySpecifyDefaultLicense() internal EndpointConfiguration EndpointConfiguration { get; } - internal ServerlessTransport MakeServerless() + internal ServerlessTransport MakeServerless() { - var serverlessTransport = new ServerlessTransport(Transport); + var serverlessTransport = new ServerlessTransport(Transport); EndpointConfiguration.UseTransport(serverlessTransport); return serverlessTransport; diff --git a/src/NServiceBus.AwsLambda.SQS/TransportWrapper/ServerlessTransport.cs b/src/NServiceBus.AwsLambda.SQS/TransportWrapper/ServerlessTransport.cs index 7535171..a2e9b44 100644 --- a/src/NServiceBus.AwsLambda.SQS/TransportWrapper/ServerlessTransport.cs +++ b/src/NServiceBus.AwsLambda.SQS/TransportWrapper/ServerlessTransport.cs @@ -6,19 +6,18 @@ using NServiceBus; using Transport; - class ServerlessTransport : TransportDefinition - where TBaseTransport : TransportDefinition + class ServerlessTransport : TransportDefinition { // HINT: This constant is defined in NServiceBus but is not exposed const string MainReceiverId = "Main"; - public ServerlessTransport(TBaseTransport baseTransport) + public ServerlessTransport(TransportDefinition baseTransport) : base(baseTransport.TransportTransactionMode, baseTransport.SupportsDelayedDelivery, baseTransport.SupportsPublishSubscribe, baseTransport.SupportsTTBR) { BaseTransport = baseTransport; } - public TBaseTransport BaseTransport { get; } + public TransportDefinition BaseTransport { get; } public PipelineInvoker PipelineInvoker { get; private set; } @@ -26,7 +25,7 @@ public override async Task Initialize(HostSettings host { // hostSettings. var baseTransportInfrastructure = await BaseTransport.Initialize(hostSettings, receivers, sendingAddresses, cancellationToken).ConfigureAwait(false); - var serverlessTransportInfrastructure = new ServerlessTransportInfrastructure(baseTransportInfrastructure); + var serverlessTransportInfrastructure = new ServerlessTransportInfrastructure(baseTransportInfrastructure); PipelineInvoker = (PipelineInvoker)serverlessTransportInfrastructure.Receivers[MainReceiverId]; return serverlessTransportInfrastructure; diff --git a/src/NServiceBus.AwsLambda.SQS/TransportWrapper/ServerlessTransportInfrastructure.cs b/src/NServiceBus.AwsLambda.SQS/TransportWrapper/ServerlessTransportInfrastructure.cs index b67e3b8..c2e3f32 100644 --- a/src/NServiceBus.AwsLambda.SQS/TransportWrapper/ServerlessTransportInfrastructure.cs +++ b/src/NServiceBus.AwsLambda.SQS/TransportWrapper/ServerlessTransportInfrastructure.cs @@ -5,7 +5,7 @@ using System.Threading.Tasks; using Transport; - class ServerlessTransportInfrastructure : TransportInfrastructure where TBaseInfrastructure : TransportDefinition + class ServerlessTransportInfrastructure : TransportInfrastructure { public ServerlessTransportInfrastructure(TransportInfrastructure baseTransportInfrastructure) { From 29ef1fdeeb8ae16f07212095a5fc79173b0eabf8 Mon Sep 17 00:00:00 2001 From: kentdr Date: Mon, 27 Mar 2023 16:16:45 -0400 Subject: [PATCH 11/27] Throw when cancellation token is cancelled. --- src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs b/src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs index 739adb1..3429e9f 100644 --- a/src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs +++ b/src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs @@ -229,7 +229,7 @@ async Task ProcessMessage(SQSEvent.SQSMessage receivedMessage, ILambdaContext la } catch (OperationCanceledException) { - return; + throw; } catch (Exception ex) { @@ -288,13 +288,15 @@ static bool IsMessageExpired(SQSEvent.SQSMessage receivedMessage, Dictionary headers, string nativeMessageId, ReadOnlyMemory body, ILambdaContext lambdaContext, CancellationToken token) { var immediateProcessingAttempts = 0; - var messageProcessedOk = false; var errorHandled = false; + var messageProcessedOk = false; while (!errorHandled && !messageProcessedOk) { try { + token.ThrowIfCancellationRequested(); + var messageContext = new MessageContext( nativeMessageId, new Dictionary(headers), @@ -305,8 +307,7 @@ async Task ProcessMessageWithInMemoryRetries(Dictionary headers, await Process(messageContext, lambdaContext, token).ConfigureAwait(false); - messageProcessedOk = !token.IsCancellationRequested; - + messageProcessedOk = true; } catch (Exception ex) when (!(ex is OperationCanceledException && token.IsCancellationRequested)) @@ -410,7 +411,7 @@ async Task MovePoisonMessageToErrorQueue(SQSEvent.SQSMessage message, string mes Logger.Warn($"Error returning poison message back to input queue at url {queueUrl}. Poison message will become available at the input queue again after the visibility timeout expires.", changeMessageVisibilityEx); } - return; + throw; } try From 1046f40a95092a80bd256f5360d8f595c8e4ef06 Mon Sep 17 00:00:00 2001 From: kentdr Date: Tue, 28 Mar 2023 16:09:30 -0400 Subject: [PATCH 12/27] Remove preview info log --- src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs b/src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs index 3429e9f..3c5c27c 100644 --- a/src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs +++ b/src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs @@ -63,7 +63,6 @@ async Task InitializeEndpointIfNecessary(ILambdaContext executionContext, Cancel var serverlessTransport = configuration.MakeServerless(); await Initialize(configuration).ConfigureAwait(false); - LogManager.GetLogger("Previews").Info("NServiceBus.AwsLambda.SQS is a preview package. Preview packages are licensed separately from the rest of the Particular Software platform and have different support guarantees. You can view the license at https://particular.net/eula/previews and the support policy at https://docs.particular.net/previews/support-policy. Customer adoption drives whether NServiceBus.AwsLambda.SQS will be incorporated into the Particular Software platform. Let us know you are using it, if you haven't already, by emailing us at support@particular.net."); endpoint = await Endpoint.Start(configuration.EndpointConfiguration, token).ConfigureAwait(false); From 91186813c7b12b6b4631caeacec4b96a422d1953 Mon Sep 17 00:00:00 2001 From: Dan Kent <83468000+kentdr@users.noreply.github.com> Date: Tue, 28 Mar 2023 16:18:39 -0400 Subject: [PATCH 13/27] Update src/NServiceBus.AwsLambda.SQS/AwsLambdaSQSEndpointConfiguration.cs Co-authored-by: Tim Bussmann --- .../AwsLambdaSQSEndpointConfiguration.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/NServiceBus.AwsLambda.SQS/AwsLambdaSQSEndpointConfiguration.cs b/src/NServiceBus.AwsLambda.SQS/AwsLambdaSQSEndpointConfiguration.cs index bea0e21..76e7b63 100644 --- a/src/NServiceBus.AwsLambda.SQS/AwsLambdaSQSEndpointConfiguration.cs +++ b/src/NServiceBus.AwsLambda.SQS/AwsLambdaSQSEndpointConfiguration.cs @@ -37,7 +37,7 @@ public AwsLambdaSQSEndpointConfiguration(string endpointName, IAmazonSQS sqsClie recoverabilityPolicy.SendFailedMessagesToErrorQueue = true; - //make sure a call to "onError" will move the message to the error queue. + // delayed delivery is disabled by default as the required FIFO queue might not exist EndpointConfiguration.Recoverability() .Delayed(c => c.NumberOfRetries(0)) .CustomPolicy(recoverabilityPolicy.Invoke); From 1035562b049b224239233365952a55911333e376 Mon Sep 17 00:00:00 2001 From: kentdr Date: Wed, 29 Mar 2023 13:51:30 -0400 Subject: [PATCH 14/27] Use clock skew from SQS Client --- src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs b/src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs index 3c5c27c..17e53e7 100644 --- a/src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs +++ b/src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs @@ -6,7 +6,6 @@ using System.Threading.Tasks; using Amazon.Lambda.Core; using Amazon.Lambda.SQSEvents; - using Amazon.Runtime; using Amazon.S3; using Amazon.SQS; using Amazon.SQS.Model; @@ -175,7 +174,6 @@ async Task Initialize(AwsLambdaSQSEndpointConfiguration configuration) var settingsHolder = configuration.AdvancedConfiguration.GetSettings(); sqsClient = configuration.Transport.SqsClient; - awsEndpointUrl = sqsClient.Config.DetermineServiceURL(); queueUrl = await GetQueueUrl(settingsHolder.EndpointName()).ConfigureAwait(false); errorQueueUrl = await GetQueueUrl(settingsHolder.ErrorQueueAddress()).ConfigureAwait(false); @@ -190,14 +188,14 @@ async Task Initialize(AwsLambdaSQSEndpointConfiguration configuration) async Task GetQueueUrl(string queueName) { - var sanitizedErrorQueueName = QueueNameHelper.GetSanitizedQueueName(queueName); + var sanitizedQueueName = QueueNameHelper.GetSanitizedQueueName(queueName); try { - return (await sqsClient.GetQueueUrlAsync(sanitizedErrorQueueName).ConfigureAwait(false)).QueueUrl; + return (await sqsClient.GetQueueUrlAsync(sanitizedQueueName).ConfigureAwait(false)).QueueUrl; } catch (Exception e) { - Logger.Error($"Failed to obtain the queue URL for queue {sanitizedErrorQueueName} (derived from configured name {queueName}).", e); + Logger.Error($"Failed to obtain the queue URL for queue {sanitizedQueueName} (derived from configured name {queueName}).", e); throw; } } @@ -245,7 +243,7 @@ async Task ProcessMessage(SQSEvent.SQSMessage receivedMessage, ILambdaContext la return; } - if (!IsMessageExpired(receivedMessage, transportMessage.Headers, messageId, CorrectClockSkew.GetClockCorrectionForEndpoint(awsEndpointUrl))) + if (!IsMessageExpired(receivedMessage, transportMessage.Headers, messageId, sqsClient.Config.ClockOffset)) { // here we also want to use the native message id because the core demands it like that await ProcessMessageWithInMemoryRetries(transportMessage.Headers, nativeMessageId, messageBody, lambdaContext, token).ConfigureAwait(false); @@ -450,7 +448,6 @@ static void LogPoisonMessage(string messageId, Exception exception) IAmazonSQS sqsClient; IAmazonS3 s3Client; string s3BucketForLargeMessages; - string awsEndpointUrl; string queueUrl; string errorQueueUrl; From 511422d701d6fa127aef7ad71576257169d446ab Mon Sep 17 00:00:00 2001 From: kentdr Date: Wed, 29 Mar 2023 14:53:16 -0400 Subject: [PATCH 15/27] Create delay queue during test startup --- .../AwsLambdaSQSEndpointTestBase.cs | 18 ++++++++++++++++++ .../When_a_handler_sends_a_message.cs | 7 +++++-- .../NServiceBus.AwsLambda.SQS.csproj | 2 +- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/AwsLambdaSQSEndpointTestBase.cs b/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/AwsLambdaSQSEndpointTestBase.cs index ba7d2f9..9691cf5 100644 --- a/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/AwsLambdaSQSEndpointTestBase.cs +++ b/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/AwsLambdaSQSEndpointTestBase.cs @@ -2,6 +2,7 @@ { using System; using System.Collections.Generic; + using System.Globalization; using System.IO; using System.Linq; using System.Threading.Tasks; @@ -17,7 +18,13 @@ [TestFixture] class AwsLambdaSQSEndpointTestBase { + protected const string DelayedDeliveryQueueSuffix = "-delay.fifo"; + const int QueueDelayInSeconds = 900; // 15 * 60 + protected string QueueName { get; set; } + + protected string DelayQueueName { get; set; } + protected string ErrorQueueName { get; set; } protected string QueueNamePrefix { get; set; } @@ -52,6 +59,17 @@ public async Task Setup() } }); RegisterQueueNameToCleanup(ErrorQueueName); + DelayQueueName = $"{QueueName}{DelayedDeliveryQueueSuffix}"; + _ = await sqsClient.CreateQueueAsync(new CreateQueueRequest(DelayQueueName) + { + Attributes = new Dictionary + { + { "FifoQueue", "true" }, + { QueueAttributeName.DelaySeconds, QueueDelayInSeconds.ToString(CultureInfo.InvariantCulture)} + } + }); + RegisterQueueNameToCleanup(DelayQueueName); + s3Client = CreateS3Client(); KeyPrefix = QueueNamePrefix; } diff --git a/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/When_a_handler_sends_a_message.cs b/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/When_a_handler_sends_a_message.cs index 806e574..d53b2a4 100644 --- a/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/When_a_handler_sends_a_message.cs +++ b/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/When_a_handler_sends_a_message.cs @@ -15,14 +15,17 @@ public async Task The_message_should_be_received() var destinationEndpointName = $"{QueueNamePrefix}DestinationEndpoint"; RegisterQueueNameToCleanup(destinationEndpointName); + RegisterQueueNameToCleanup(destinationEndpointName + DelayedDeliveryQueueSuffix); var destinationConfiguration = new EndpointConfiguration(destinationEndpointName); - var destinationTransport = destinationConfiguration.UseTransport(); - destinationTransport.ClientFactory(CreateSQSClient); + var destinationTransport = new SqsTransport(CreateSQSClient(), CreateSNSClient()); + destinationConfiguration.SendFailedMessagesTo(ErrorQueueName); destinationConfiguration.EnableInstallers(); destinationConfiguration.RegisterComponents(c => c.AddSingleton(typeof(TestContext), context)); + destinationConfiguration.UseTransport(destinationTransport); + var destinationEndpoint = await Endpoint.Start(destinationConfiguration); var endpoint = new AwsLambdaSQSEndpoint(ctx => diff --git a/src/NServiceBus.AwsLambda.SQS/NServiceBus.AwsLambda.SQS.csproj b/src/NServiceBus.AwsLambda.SQS/NServiceBus.AwsLambda.SQS.csproj index 7671bc2..4214add 100644 --- a/src/NServiceBus.AwsLambda.SQS/NServiceBus.AwsLambda.SQS.csproj +++ b/src/NServiceBus.AwsLambda.SQS/NServiceBus.AwsLambda.SQS.csproj @@ -9,7 +9,7 @@ - + From ef47da89fd51cd3c66338cf4409616bed38aa3cc Mon Sep 17 00:00:00 2001 From: kentdr Date: Wed, 29 Mar 2023 15:02:00 -0400 Subject: [PATCH 16/27] Remove TransactionMode property from ServerlessTransportInfrastructure --- .../TransportWrapper/ServerlessTransportInfrastructure.cs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/NServiceBus.AwsLambda.SQS/TransportWrapper/ServerlessTransportInfrastructure.cs b/src/NServiceBus.AwsLambda.SQS/TransportWrapper/ServerlessTransportInfrastructure.cs index c2e3f32..fd9b43e 100644 --- a/src/NServiceBus.AwsLambda.SQS/TransportWrapper/ServerlessTransportInfrastructure.cs +++ b/src/NServiceBus.AwsLambda.SQS/TransportWrapper/ServerlessTransportInfrastructure.cs @@ -14,10 +14,6 @@ public ServerlessTransportInfrastructure(TransportInfrastructure baseTransportIn Receivers = baseTransportInfrastructure.Receivers.ToDictionary(r => r.Key, r => (IMessageReceiver)new PipelineInvoker(r.Value)); } - //TODO: Is this still necessary? - //support ReceiveOnly so that we can use immediate retries - public TransportTransactionMode TransactionMode { get; } = TransportTransactionMode.ReceiveOnly; - public override Task Shutdown(CancellationToken cancellationToken = default) => baseTransportInfrastructure.Shutdown(cancellationToken); public override string ToTransportAddress(QueueAddress address) => baseTransportInfrastructure.ToTransportAddress(address); From b3c8e429d417394efdf0978389680f839f8b29d6 Mon Sep 17 00:00:00 2001 From: kentdr Date: Wed, 29 Mar 2023 15:07:28 -0400 Subject: [PATCH 17/27] Cache error handled task --- .../TransportWrapper/PipelineInvoker.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/NServiceBus.AwsLambda.SQS/TransportWrapper/PipelineInvoker.cs b/src/NServiceBus.AwsLambda.SQS/TransportWrapper/PipelineInvoker.cs index cffd651..922b998 100644 --- a/src/NServiceBus.AwsLambda.SQS/TransportWrapper/PipelineInvoker.cs +++ b/src/NServiceBus.AwsLambda.SQS/TransportWrapper/PipelineInvoker.cs @@ -22,10 +22,11 @@ Task IMessageReceiver.Initialize(PushRuntimeSettings limitations, OnMessage onMe { this.onMessage = onMessage; this.onError = onError; + var errorHandledResultTask = Task.FromResult(ErrorHandleResult.Handled); return baseTransportReceiver?.Initialize(limitations, (_, __) => Task.CompletedTask, - (_, __) => Task.FromResult(ErrorHandleResult.Handled), + (_, __) => errorHandledResultTask, cancellationToken) ?? Task.CompletedTask; } From 9980f4838dbd23fe0715aac4200a57ac9ab94268 Mon Sep 17 00:00:00 2001 From: kentdr Date: Fri, 31 Mar 2023 10:09:38 -0400 Subject: [PATCH 18/27] Move MakeServerless out of endpoint configuration --- .../AwsLambdaEndpoint.cs | 16 ++++++++++------ .../AwsLambdaSQSEndpointConfiguration.cs | 10 +--------- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs b/src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs index 17e53e7..49a949f 100644 --- a/src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs +++ b/src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs @@ -59,9 +59,8 @@ async Task InitializeEndpointIfNecessary(ILambdaContext executionContext, Cancel if (pipeline == null) { var configuration = configurationFactory(executionContext); - var serverlessTransport = configuration.MakeServerless(); - await Initialize(configuration).ConfigureAwait(false); + var serverlessTransport = await Initialize(configuration).ConfigureAwait(false); endpoint = await Endpoint.Start(configuration.EndpointConfiguration, token).ConfigureAwait(false); @@ -169,7 +168,7 @@ public async Task Unsubscribe(Type eventType, ILambdaContext lambdaContext) await endpoint.Unsubscribe(eventType).ConfigureAwait(false); } - async Task Initialize(AwsLambdaSQSEndpointConfiguration configuration) + async Task Initialize(AwsLambdaSQSEndpointConfiguration configuration) { var settingsHolder = configuration.AdvancedConfiguration.GetSettings(); @@ -179,11 +178,16 @@ async Task Initialize(AwsLambdaSQSEndpointConfiguration configuration) errorQueueUrl = await GetQueueUrl(settingsHolder.ErrorQueueAddress()).ConfigureAwait(false); s3BucketForLargeMessages = configuration.Transport.S3?.BucketName; - if (string.IsNullOrWhiteSpace(s3BucketForLargeMessages)) + + if (!string.IsNullOrWhiteSpace(s3BucketForLargeMessages)) { - return; + s3Client = configuration.Transport.S3?.S3Client; } - s3Client = configuration.Transport.S3?.S3Client; + + var serverlessTransport = new ServerlessTransport(configuration.Transport); + configuration.EndpointConfiguration.UseTransport(serverlessTransport); + + return serverlessTransport; } async Task GetQueueUrl(string queueName) diff --git a/src/NServiceBus.AwsLambda.SQS/AwsLambdaSQSEndpointConfiguration.cs b/src/NServiceBus.AwsLambda.SQS/AwsLambdaSQSEndpointConfiguration.cs index 76e7b63..f299e8b 100644 --- a/src/NServiceBus.AwsLambda.SQS/AwsLambdaSQSEndpointConfiguration.cs +++ b/src/NServiceBus.AwsLambda.SQS/AwsLambdaSQSEndpointConfiguration.cs @@ -5,7 +5,7 @@ using Amazon.SimpleNotificationService; using Amazon.SQS; using AwsLambda.SQS; - using NServiceBus.AwsLambda.SQS.TransportWrapper; + using NServiceBus.Logging; using Serialization; @@ -83,14 +83,6 @@ void TrySpecifyDefaultLicense() internal EndpointConfiguration EndpointConfiguration { get; } - internal ServerlessTransport MakeServerless() - { - var serverlessTransport = new ServerlessTransport(Transport); - EndpointConfiguration.UseTransport(serverlessTransport); - - return serverlessTransport; - } - /// /// Gives access to the underlying endpoint configuration for advanced configuration options. /// From 64194b4e200b42a9a00f5468cbd8e1495a72c603 Mon Sep 17 00:00:00 2001 From: kentdr Date: Fri, 31 Mar 2023 10:17:23 -0400 Subject: [PATCH 19/27] Change catch block to use when condition --- src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs b/src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs index 49a949f..321a168 100644 --- a/src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs +++ b/src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs @@ -228,11 +228,7 @@ async Task ProcessMessage(SQSEvent.SQSMessage receivedMessage, ILambdaContext la messageBody = await transportMessage.RetrieveBody(s3Client, s3BucketForLargeMessages, token).ConfigureAwait(false); } - catch (OperationCanceledException) - { - throw; - } - catch (Exception ex) + catch (Exception ex) when (ex is not OperationCanceledException) { // Can't deserialize. This is a poison message exception = ex; From 06800e75a4d2f9c323398ff0de99f53a4921fea8 Mon Sep 17 00:00:00 2001 From: kentdr Date: Fri, 31 Mar 2023 10:31:12 -0400 Subject: [PATCH 20/27] Update TransportMessage ctor argument name to match sqs transport --- src/NServiceBus.AwsLambda.SQS/TransportMessage.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/NServiceBus.AwsLambda.SQS/TransportMessage.cs b/src/NServiceBus.AwsLambda.SQS/TransportMessage.cs index 2152713..a37fc26 100644 --- a/src/NServiceBus.AwsLambda.SQS/TransportMessage.cs +++ b/src/NServiceBus.AwsLambda.SQS/TransportMessage.cs @@ -11,7 +11,7 @@ public TransportMessage() { } - public TransportMessage(OutgoingMessage outgoingMessage, DispatchProperties deliveryConstraints) + public TransportMessage(OutgoingMessage outgoingMessage, DispatchProperties properties) { Headers = outgoingMessage.Headers; @@ -22,9 +22,9 @@ public TransportMessage(OutgoingMessage outgoingMessage, DispatchProperties deli Headers[NServiceBus.Headers.MessageId] = messageId; } - if (deliveryConstraints.DiscardIfNotReceivedBefore != null) + if (properties.DiscardIfNotReceivedBefore != null) { - TimeToBeReceived = deliveryConstraints.DiscardIfNotReceivedBefore.MaxTime.ToString(); + TimeToBeReceived = properties.DiscardIfNotReceivedBefore.MaxTime.ToString(); } Body = !outgoingMessage.Body.IsEmpty ? Convert.ToBase64String(outgoingMessage.Body.Span) : "empty message"; From c374a3ead56f26728c8c51ebe0335f4d76ee8448 Mon Sep 17 00:00:00 2001 From: Dan Kent <83468000+kentdr@users.noreply.github.com> Date: Fri, 31 Mar 2023 10:24:12 -0400 Subject: [PATCH 21/27] Update src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs Co-authored-by: Tim Bussmann --- src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs b/src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs index 321a168..0ace508 100644 --- a/src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs +++ b/src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs @@ -322,8 +322,7 @@ async Task ProcessMessageWithInMemoryRetries(Dictionary headers, transportTransaction, immediateProcessingAttempts, queueUrl, - new ContextBag() - ); + new ContextBag()); errorHandlerResult = await ProcessFailedMessage(errorContext, lambdaContext).ConfigureAwait(false); } From 1079d5d1f4b3a69e793ce8ed83538a9877623b85 Mon Sep 17 00:00:00 2001 From: kentdr Date: Fri, 31 Mar 2023 10:37:02 -0400 Subject: [PATCH 22/27] Return in retry loop instead of setting flag --- src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs b/src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs index 0ace508..3c7c437 100644 --- a/src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs +++ b/src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs @@ -286,9 +286,8 @@ async Task ProcessMessageWithInMemoryRetries(Dictionary headers, { var immediateProcessingAttempts = 0; var errorHandled = false; - var messageProcessedOk = false; - while (!errorHandled && !messageProcessedOk) + while (!errorHandled) { try { @@ -304,7 +303,7 @@ async Task ProcessMessageWithInMemoryRetries(Dictionary headers, await Process(messageContext, lambdaContext, token).ConfigureAwait(false); - messageProcessedOk = true; + return; } catch (Exception ex) when (!(ex is OperationCanceledException && token.IsCancellationRequested)) From df4abaee67cf797ceec8af11320ce84efce56830 Mon Sep 17 00:00:00 2001 From: danielmarbach Date: Tue, 4 Apr 2023 10:30:42 +0200 Subject: [PATCH 23/27] Cosmetics --- .../AwsLambdaEndpoint.cs | 117 +++++++++++------- .../AwsLambdaSQSEndpointConfiguration.cs | 8 +- src/NServiceBus.AwsLambda.SQS/LambdaLog.cs | 9 +- .../LambdaLoggerDefinition.cs | 12 +- .../LambdaLoggerFactory.cs | 15 +-- .../MessageExtensions.cs | 10 +- .../ServerlessRecoverabilityPolicy.cs | 16 +-- src/NServiceBus.AwsLambda.SQS/SettingsKeys.cs | 10 -- .../TransportWrapper/PipelineInvoker.cs | 33 ++--- .../TransportWrapper/ServerlessTransport.cs | 15 ++- .../ServerlessTransportInfrastructure.cs | 2 +- 11 files changed, 113 insertions(+), 134 deletions(-) delete mode 100644 src/NServiceBus.AwsLambda.SQS/SettingsKeys.cs diff --git a/src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs b/src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs index 3c7c437..64c061f 100644 --- a/src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs +++ b/src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs @@ -37,7 +37,8 @@ public AwsLambdaSQSEndpoint(Func(); @@ -46,30 +47,34 @@ public async Task Process(SQSEvent @event, ILambdaContext lambdaContext, Cancell processTasks.Add(ProcessMessage(receivedMessage, lambdaContext, cancellationToken)); } - await Task.WhenAll(processTasks).ConfigureAwait(false); + await Task.WhenAll(processTasks) + .ConfigureAwait(false); } async Task InitializeEndpointIfNecessary(ILambdaContext executionContext, CancellationToken token = default) { if (pipeline == null) { - await semaphoreLock.WaitAsync(token).ConfigureAwait(false); + await semaphoreLock.WaitAsync(token) + .ConfigureAwait(false); try { if (pipeline == null) { var configuration = configurationFactory(executionContext); - var serverlessTransport = await Initialize(configuration).ConfigureAwait(false); + var serverlessTransport = await Initialize(configuration) + .ConfigureAwait(false); - endpoint = await Endpoint.Start(configuration.EndpointConfiguration, token).ConfigureAwait(false); + endpoint = await Endpoint.Start(configuration.EndpointConfiguration, token) + .ConfigureAwait(false); pipeline = serverlessTransport.PipelineInvoker; } } finally { - semaphoreLock.Release(); + _ = semaphoreLock.Release(); } } } @@ -84,88 +89,106 @@ public async Task Send(object message, SendOptions options, ILambdaContext lambd /// public Task Send(object message, ILambdaContext lambdaContext) - { - return Send(message, new SendOptions(), lambdaContext); - } + => Send(message, new SendOptions(), lambdaContext); /// public async Task Send(Action messageConstructor, SendOptions options, ILambdaContext lambdaContext) { - await InitializeEndpointIfNecessary(lambdaContext).ConfigureAwait(false); + await InitializeEndpointIfNecessary(lambdaContext) + .ConfigureAwait(false); - await endpoint.Send(messageConstructor, options).ConfigureAwait(false); + await endpoint.Send(messageConstructor, options) + .ConfigureAwait(false); } /// public async Task Send(Action messageConstructor, ILambdaContext lambdaContext) { - await InitializeEndpointIfNecessary(lambdaContext).ConfigureAwait(false); + await InitializeEndpointIfNecessary(lambdaContext) + .ConfigureAwait(false); - await Send(messageConstructor, new SendOptions(), lambdaContext).ConfigureAwait(false); + await Send(messageConstructor, new SendOptions(), lambdaContext) + .ConfigureAwait(false); } /// public async Task Publish(object message, PublishOptions options, ILambdaContext lambdaContext) { - await InitializeEndpointIfNecessary(lambdaContext).ConfigureAwait(false); + await InitializeEndpointIfNecessary(lambdaContext) + .ConfigureAwait(false); - await endpoint.Publish(message, options).ConfigureAwait(false); + await endpoint.Publish(message, options) + .ConfigureAwait(false); } /// public async Task Publish(Action messageConstructor, PublishOptions options, ILambdaContext lambdaContext) { - await InitializeEndpointIfNecessary(lambdaContext).ConfigureAwait(false); + await InitializeEndpointIfNecessary(lambdaContext) + .ConfigureAwait(false); - await endpoint.Publish(messageConstructor, options).ConfigureAwait(false); + await endpoint.Publish(messageConstructor, options) + .ConfigureAwait(false); } /// public async Task Publish(object message, ILambdaContext lambdaContext) { - await InitializeEndpointIfNecessary(lambdaContext).ConfigureAwait(false); + await InitializeEndpointIfNecessary(lambdaContext) + .ConfigureAwait(false); - await endpoint.Publish(message).ConfigureAwait(false); + await endpoint.Publish(message) + .ConfigureAwait(false); } /// public async Task Publish(Action messageConstructor, ILambdaContext lambdaContext) { - await InitializeEndpointIfNecessary(lambdaContext).ConfigureAwait(false); + await InitializeEndpointIfNecessary(lambdaContext) + .ConfigureAwait(false); - await endpoint.Publish(messageConstructor).ConfigureAwait(false); + await endpoint.Publish(messageConstructor) + .ConfigureAwait(false); } /// public async Task Subscribe(Type eventType, SubscribeOptions options, ILambdaContext lambdaContext) { - await InitializeEndpointIfNecessary(lambdaContext).ConfigureAwait(false); + await InitializeEndpointIfNecessary(lambdaContext) + .ConfigureAwait(false); - await endpoint.Subscribe(eventType, options).ConfigureAwait(false); + await endpoint.Subscribe(eventType, options) + .ConfigureAwait(false); } /// public async Task Subscribe(Type eventType, ILambdaContext lambdaContext) { - await InitializeEndpointIfNecessary(lambdaContext).ConfigureAwait(false); + await InitializeEndpointIfNecessary(lambdaContext) + .ConfigureAwait(false); - await endpoint.Subscribe(eventType).ConfigureAwait(false); + await endpoint.Subscribe(eventType) + .ConfigureAwait(false); } /// public async Task Unsubscribe(Type eventType, UnsubscribeOptions options, ILambdaContext lambdaContext) { - await InitializeEndpointIfNecessary(lambdaContext).ConfigureAwait(false); + await InitializeEndpointIfNecessary(lambdaContext) + .ConfigureAwait(false); - await endpoint.Unsubscribe(eventType, options).ConfigureAwait(false); + await endpoint.Unsubscribe(eventType, options) + .ConfigureAwait(false); } /// public async Task Unsubscribe(Type eventType, ILambdaContext lambdaContext) { - await InitializeEndpointIfNecessary(lambdaContext).ConfigureAwait(false); + await InitializeEndpointIfNecessary(lambdaContext) + .ConfigureAwait(false); - await endpoint.Unsubscribe(eventType).ConfigureAwait(false); + await endpoint.Unsubscribe(eventType) + .ConfigureAwait(false); } async Task Initialize(AwsLambdaSQSEndpointConfiguration configuration) @@ -174,8 +197,10 @@ async Task Initialize(AwsLambdaSQSEndpointConfiguration con sqsClient = configuration.Transport.SqsClient; - queueUrl = await GetQueueUrl(settingsHolder.EndpointName()).ConfigureAwait(false); - errorQueueUrl = await GetQueueUrl(settingsHolder.ErrorQueueAddress()).ConfigureAwait(false); + queueUrl = await GetQueueUrl(settingsHolder.EndpointName()) + .ConfigureAwait(false); + errorQueueUrl = await GetQueueUrl(settingsHolder.ErrorQueueAddress()) + .ConfigureAwait(false); s3BucketForLargeMessages = configuration.Transport.S3?.BucketName; @@ -338,15 +363,19 @@ async Task ProcessMessageWithInMemoryRetries(Dictionary headers, async Task Process(MessageContext messageContext, ILambdaContext executionContext, CancellationToken cancellationToken) { - await InitializeEndpointIfNecessary(executionContext, cancellationToken).ConfigureAwait(false); - await pipeline.PushMessage(messageContext).ConfigureAwait(false); + await InitializeEndpointIfNecessary(executionContext, cancellationToken) + .ConfigureAwait(false); + await pipeline.PushMessage(messageContext) + .ConfigureAwait(false); } async Task ProcessFailedMessage(ErrorContext errorContext, ILambdaContext executionContext) { - await InitializeEndpointIfNecessary(executionContext).ConfigureAwait(false); + await InitializeEndpointIfNecessary(executionContext) + .ConfigureAwait(false); - return await pipeline.PushFailedMessage(errorContext).ConfigureAwait(false); + return await pipeline.PushFailedMessage(errorContext) + .ConfigureAwait(false); } async Task DeleteMessageAndBodyIfRequired(SQSEvent.SQSMessage message, string messageS3BodyKey) @@ -354,7 +383,8 @@ async Task DeleteMessageAndBodyIfRequired(SQSEvent.SQSMessage message, string me try { // should not be cancelled - await sqsClient.DeleteMessageAsync(queueUrl, message.ReceiptHandle, CancellationToken.None).ConfigureAwait(false); + await sqsClient.DeleteMessageAsync(queueUrl, message.ReceiptHandle, CancellationToken.None) + .ConfigureAwait(false); } catch (ReceiptHandleIsInvalidException ex) { @@ -384,7 +414,8 @@ async Task MovePoisonMessageToErrorQueue(SQSEvent.SQSMessage message, string mes DataType = "String" } } - }, CancellationToken.None).ConfigureAwait(false); + }, CancellationToken.None) + .ConfigureAwait(false); // The MessageAttributes on message are read-only attributes provided by SQS // and can't be re-sent. Unfortunately all the SQS metadata // such as SentTimestamp is reset with this send. @@ -399,7 +430,8 @@ async Task MovePoisonMessageToErrorQueue(SQSEvent.SQSMessage message, string mes QueueUrl = queueUrl, ReceiptHandle = message.ReceiptHandle, VisibilityTimeout = 0 - }, CancellationToken.None).ConfigureAwait(false); + }, CancellationToken.None) + .ConfigureAwait(false); } catch (Exception changeMessageVisibilityEx) { @@ -415,7 +447,8 @@ async Task MovePoisonMessageToErrorQueue(SQSEvent.SQSMessage message, string mes { QueueUrl = queueUrl, ReceiptHandle = message.ReceiptHandle - }, CancellationToken.None).ConfigureAwait(false); + }, CancellationToken.None) + .ConfigureAwait(false); } catch (Exception ex) { @@ -440,7 +473,7 @@ static void LogPoisonMessage(string messageId, Exception exception) } readonly Func configurationFactory; - readonly SemaphoreSlim semaphoreLock = new SemaphoreSlim(initialCount: 1, maxCount: 1); + readonly SemaphoreSlim semaphoreLock = new(initialCount: 1, maxCount: 1); PipelineInvoker pipeline; IEndpointInstance endpoint; IAmazonSQS sqsClient; @@ -449,7 +482,7 @@ static void LogPoisonMessage(string messageId, Exception exception) string queueUrl; string errorQueueUrl; - static ILog Logger = LogManager.GetLogger(typeof(AwsLambdaSQSEndpoint)); - static readonly TransportTransaction transportTransaction = new TransportTransaction(); + static readonly ILog Logger = LogManager.GetLogger(typeof(AwsLambdaSQSEndpoint)); + static readonly TransportTransaction transportTransaction = new(); } } \ No newline at end of file diff --git a/src/NServiceBus.AwsLambda.SQS/AwsLambdaSQSEndpointConfiguration.cs b/src/NServiceBus.AwsLambda.SQS/AwsLambdaSQSEndpointConfiguration.cs index f299e8b..72089b7 100644 --- a/src/NServiceBus.AwsLambda.SQS/AwsLambdaSQSEndpointConfiguration.cs +++ b/src/NServiceBus.AwsLambda.SQS/AwsLambdaSQSEndpointConfiguration.cs @@ -92,16 +92,12 @@ void TrySpecifyDefaultLicense() /// Define the serializer to be used. /// public SerializationExtensions UseSerialization() where T : SerializationDefinition, new() - { - return EndpointConfiguration.UseSerialization(); - } + => EndpointConfiguration.UseSerialization(); /// /// Disables moving messages to the error queue even if an error queue name is configured. /// public void DoNotSendMessagesToErrorQueue() - { - recoverabilityPolicy.SendFailedMessagesToErrorQueue = false; - } + => recoverabilityPolicy.SendFailedMessagesToErrorQueue = false; } } \ No newline at end of file diff --git a/src/NServiceBus.AwsLambda.SQS/LambdaLog.cs b/src/NServiceBus.AwsLambda.SQS/LambdaLog.cs index 85b3466..b47ef02 100644 --- a/src/NServiceBus.AwsLambda.SQS/LambdaLog.cs +++ b/src/NServiceBus.AwsLambda.SQS/LambdaLog.cs @@ -27,13 +27,10 @@ public LambdaLog(string name, LogLevel level) } void Write(string level, string message, Exception exception) - { - LambdaLogger.Log($"{name}. {level}. {message}. Exception: {exception}"); - } + => LambdaLogger.Log($"{name}. {level}. {message}. Exception: {exception}"); + void Write(string level, string message) - { - LambdaLogger.Log($"{name}. {level}. {message}."); - } + => LambdaLogger.Log($"{name}. {level}. {message}."); void Write(string level, string format, params object[] args) { diff --git a/src/NServiceBus.AwsLambda.SQS/LambdaLoggerDefinition.cs b/src/NServiceBus.AwsLambda.SQS/LambdaLoggerDefinition.cs index e4de5c4..0c862f5 100644 --- a/src/NServiceBus.AwsLambda.SQS/LambdaLoggerDefinition.cs +++ b/src/NServiceBus.AwsLambda.SQS/LambdaLoggerDefinition.cs @@ -4,16 +4,10 @@ class LambdaLoggerDefinition : LoggingFactoryDefinition { - LogLevel level = LogLevel.Info; + public void Level(LogLevel level) => this.level = level; - public void Level(LogLevel level) - { - this.level = level; - } + protected override ILoggerFactory GetLoggingFactory() => new LambdaLoggerFactory(level); - protected override ILoggerFactory GetLoggingFactory() - { - return new LambdaLoggerFactory(level); - } + LogLevel level = LogLevel.Info; } } \ No newline at end of file diff --git a/src/NServiceBus.AwsLambda.SQS/LambdaLoggerFactory.cs b/src/NServiceBus.AwsLambda.SQS/LambdaLoggerFactory.cs index da7017c..ca3ff31 100644 --- a/src/NServiceBus.AwsLambda.SQS/LambdaLoggerFactory.cs +++ b/src/NServiceBus.AwsLambda.SQS/LambdaLoggerFactory.cs @@ -7,19 +7,10 @@ class LambdaLoggerFactory : ILoggerFactory { LogLevel level; - public LambdaLoggerFactory(LogLevel level) - { - this.level = level; - } + public LambdaLoggerFactory(LogLevel level) => this.level = level; - public ILog GetLogger(Type type) - { - return GetLogger(type.FullName); - } + public ILog GetLogger(Type type) => GetLogger(type.FullName); - public ILog GetLogger(string name) - { - return new LambdaLog(name, level); - } + public ILog GetLogger(string name) => new LambdaLog(name, level); } } \ No newline at end of file diff --git a/src/NServiceBus.AwsLambda.SQS/MessageExtensions.cs b/src/NServiceBus.AwsLambda.SQS/MessageExtensions.cs index a17121b..16ed0e3 100644 --- a/src/NServiceBus.AwsLambda.SQS/MessageExtensions.cs +++ b/src/NServiceBus.AwsLambda.SQS/MessageExtensions.cs @@ -29,11 +29,9 @@ static class MessageExtensions transportMessage.S3BodyKey, cancellationToken).ConfigureAwait(false); - using (var memoryStream = new MemoryStream()) - { - await s3GetResponse.ResponseStream.CopyToAsync(memoryStream, cancellationToken).ConfigureAwait(false); - return memoryStream.ToArray(); - } + using var memoryStream = new MemoryStream(); + await s3GetResponse.ResponseStream.CopyToAsync(memoryStream, cancellationToken).ConfigureAwait(false); + return memoryStream.ToArray(); } public static DateTime GetAdjustedDateTimeFromServerSetAttributes(this SQSEvent.SQSMessage message, TimeSpan clockOffset) @@ -44,6 +42,6 @@ public static DateTime GetAdjustedDateTimeFromServerSetAttributes(this SQSEvent. return result + clockOffset; } - static readonly DateTime UnixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); + static readonly DateTime UnixEpoch = new(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); } } \ No newline at end of file diff --git a/src/NServiceBus.AwsLambda.SQS/ServerlessRecoverabilityPolicy.cs b/src/NServiceBus.AwsLambda.SQS/ServerlessRecoverabilityPolicy.cs index 2c43781..b888ae5 100644 --- a/src/NServiceBus.AwsLambda.SQS/ServerlessRecoverabilityPolicy.cs +++ b/src/NServiceBus.AwsLambda.SQS/ServerlessRecoverabilityPolicy.cs @@ -11,18 +11,18 @@ public RecoverabilityAction Invoke(RecoverabilityConfig config, ErrorContext err { var action = DefaultRecoverabilityPolicy.Invoke(config, errorContext); - if (action is MoveToError) + if (action is not MoveToError) { - if (SendFailedMessagesToErrorQueue) - { - return action; - } + return action; + } - // 7.2 offers a Discard option, but we want to bubble up the exception so it can fail the function invocation. - throw new Exception("Failed to process message.", errorContext.Exception); + if (SendFailedMessagesToErrorQueue) + { + return action; } - return action; + // 7.2 offers a Discard option, but we want to bubble up the exception so it can fail the function invocation. + throw new Exception("Failed to process message.", errorContext.Exception); } } } \ No newline at end of file diff --git a/src/NServiceBus.AwsLambda.SQS/SettingsKeys.cs b/src/NServiceBus.AwsLambda.SQS/SettingsKeys.cs deleted file mode 100644 index 7d20972..0000000 --- a/src/NServiceBus.AwsLambda.SQS/SettingsKeys.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace NServiceBus.AwsLambda.SQS -{ - static class SettingsKeys - { - const string Prefix = "NServiceBus.AmazonSQS."; - public const string SqsClientFactory = Prefix + nameof(SqsClientFactory); - public const string S3ClientFactory = Prefix + nameof(S3ClientFactory); - public const string S3BucketForLargeMessages = Prefix + nameof(S3BucketForLargeMessages); - } -} \ No newline at end of file diff --git a/src/NServiceBus.AwsLambda.SQS/TransportWrapper/PipelineInvoker.cs b/src/NServiceBus.AwsLambda.SQS/TransportWrapper/PipelineInvoker.cs index 922b998..1941fd1 100644 --- a/src/NServiceBus.AwsLambda.SQS/TransportWrapper/PipelineInvoker.cs +++ b/src/NServiceBus.AwsLambda.SQS/TransportWrapper/PipelineInvoker.cs @@ -4,16 +4,12 @@ using System.Threading.Tasks; using Transport; - class PipelineInvoker : IMessageReceiver + sealed class PipelineInvoker : IMessageReceiver { - public PipelineInvoker(IMessageReceiver baseTransportReceiver) - { - this.baseTransportReceiver = baseTransportReceiver; - } + public PipelineInvoker(IMessageReceiver baseTransportReceiver) => this.baseTransportReceiver = baseTransportReceiver; public ISubscriptionManager Subscriptions => baseTransportReceiver.Subscriptions; - public string Id => baseTransportReceiver.Id; public string ReceiveAddress => baseTransportReceiver.ReceiveAddress; @@ -30,30 +26,15 @@ Task IMessageReceiver.Initialize(PushRuntimeSettings limitations, OnMessage onMe cancellationToken) ?? Task.CompletedTask; } - Task IMessageReceiver.StartReceive(CancellationToken cancellationToken) - { - return Task.CompletedTask; - } + Task IMessageReceiver.StartReceive(CancellationToken cancellationToken) => Task.CompletedTask; - Task IMessageReceiver.StopReceive(CancellationToken cancellationToken) - { - return Task.CompletedTask; - } + Task IMessageReceiver.StopReceive(CancellationToken cancellationToken) => Task.CompletedTask; - public Task PushFailedMessage(ErrorContext errorContext) - { - return onError(errorContext); - } + public Task PushFailedMessage(ErrorContext errorContext) => onError(errorContext); - Task IMessageReceiver.ChangeConcurrency(PushRuntimeSettings limitations, CancellationToken cancellationToken) - { - return Task.CompletedTask; - } + Task IMessageReceiver.ChangeConcurrency(PushRuntimeSettings limitations, CancellationToken cancellationToken) => Task.CompletedTask; - public Task PushMessage(MessageContext messageContext) - { - return onMessage.Invoke(messageContext); - } + public Task PushMessage(MessageContext messageContext) => onMessage.Invoke(messageContext); OnMessage onMessage; OnError onError; diff --git a/src/NServiceBus.AwsLambda.SQS/TransportWrapper/ServerlessTransport.cs b/src/NServiceBus.AwsLambda.SQS/TransportWrapper/ServerlessTransport.cs index a2e9b44..d347c02 100644 --- a/src/NServiceBus.AwsLambda.SQS/TransportWrapper/ServerlessTransport.cs +++ b/src/NServiceBus.AwsLambda.SQS/TransportWrapper/ServerlessTransport.cs @@ -6,16 +6,14 @@ using NServiceBus; using Transport; - class ServerlessTransport : TransportDefinition + sealed class ServerlessTransport : TransportDefinition { // HINT: This constant is defined in NServiceBus but is not exposed const string MainReceiverId = "Main"; public ServerlessTransport(TransportDefinition baseTransport) - : base(baseTransport.TransportTransactionMode, baseTransport.SupportsDelayedDelivery, baseTransport.SupportsPublishSubscribe, baseTransport.SupportsTTBR) - { + : base(baseTransport.TransportTransactionMode, baseTransport.SupportsDelayedDelivery, baseTransport.SupportsPublishSubscribe, baseTransport.SupportsTTBR) => BaseTransport = baseTransport; - } public TransportDefinition BaseTransport { get; } @@ -23,8 +21,9 @@ public ServerlessTransport(TransportDefinition baseTransport) public override async Task Initialize(HostSettings hostSettings, ReceiveSettings[] receivers, string[] sendingAddresses, CancellationToken cancellationToken = default) { - // hostSettings. - var baseTransportInfrastructure = await BaseTransport.Initialize(hostSettings, receivers, sendingAddresses, cancellationToken).ConfigureAwait(false); + var baseTransportInfrastructure = await BaseTransport.Initialize(hostSettings, receivers, sendingAddresses, cancellationToken) + .ConfigureAwait(false); + var serverlessTransportInfrastructure = new ServerlessTransportInfrastructure(baseTransportInfrastructure); PipelineInvoker = (PipelineInvoker)serverlessTransportInfrastructure.Receivers[MainReceiverId]; @@ -33,11 +32,11 @@ public override async Task Initialize(HostSettings host } #pragma warning disable CS0672 // Member overrides obsolete member -#pragma warning disable CS0618 // Type or memeber is obsolete +#pragma warning disable CS0618 // Type or member is obsolete public override string ToTransportAddress(QueueAddress address) => BaseTransport.ToTransportAddress(address); -#pragma warning restore CS0618 // Type or memeber is obsolete +#pragma warning restore CS0618 // Type or member is obsolete #pragma warning restore CS0672 // Member overrides obsolete member diff --git a/src/NServiceBus.AwsLambda.SQS/TransportWrapper/ServerlessTransportInfrastructure.cs b/src/NServiceBus.AwsLambda.SQS/TransportWrapper/ServerlessTransportInfrastructure.cs index fd9b43e..131db87 100644 --- a/src/NServiceBus.AwsLambda.SQS/TransportWrapper/ServerlessTransportInfrastructure.cs +++ b/src/NServiceBus.AwsLambda.SQS/TransportWrapper/ServerlessTransportInfrastructure.cs @@ -5,7 +5,7 @@ using System.Threading.Tasks; using Transport; - class ServerlessTransportInfrastructure : TransportInfrastructure + sealed class ServerlessTransportInfrastructure : TransportInfrastructure { public ServerlessTransportInfrastructure(TransportInfrastructure baseTransportInfrastructure) { From 738fd457b2dbc75ec5602645ef0a385dc6501397 Mon Sep 17 00:00:00 2001 From: danielmarbach Date: Tue, 4 Apr 2023 10:40:25 +0200 Subject: [PATCH 24/27] Switch to System.Text.Json --- .../AwsLambdaEndpoint.cs | 8 +- src/NServiceBus.AwsLambda.SQS/SimpleJson.g.cs | 2150 ----------------- .../TransportMessage.cs | 16 +- .../TransportMessageSerializerContext.cs | 10 + 4 files changed, 31 insertions(+), 2153 deletions(-) delete mode 100644 src/NServiceBus.AwsLambda.SQS/SimpleJson.g.cs create mode 100644 src/NServiceBus.AwsLambda.SQS/TransportMessageSerializerContext.cs diff --git a/src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs b/src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs index 64c061f..8175cb3 100644 --- a/src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs +++ b/src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs @@ -2,6 +2,7 @@ { using System; using System.Collections.Generic; + using System.Text.Json; using System.Threading; using System.Threading.Tasks; using Amazon.Lambda.Core; @@ -14,7 +15,6 @@ using Configuration.AdvancedExtensibility; using Extensibility; using Logging; - using SimpleJson; using Transport; /// @@ -249,7 +249,7 @@ async Task ProcessMessage(SQSEvent.SQSMessage receivedMessage, ILambdaContext la messageId = nativeMessageId; } - transportMessage = SimpleJson.DeserializeObject(receivedMessage.Body); + transportMessage = JsonSerializer.Deserialize(receivedMessage.Body, transportMessageSerializerOptions); messageBody = await transportMessage.RetrieveBody(s3Client, s3BucketForLargeMessages, token).ConfigureAwait(false); } @@ -474,6 +474,10 @@ static void LogPoisonMessage(string messageId, Exception exception) readonly Func configurationFactory; readonly SemaphoreSlim semaphoreLock = new(initialCount: 1, maxCount: 1); + readonly JsonSerializerOptions transportMessageSerializerOptions = new() + { + TypeInfoResolver = TransportMessageSerializerContext.Default + }; PipelineInvoker pipeline; IEndpointInstance endpoint; IAmazonSQS sqsClient; diff --git a/src/NServiceBus.AwsLambda.SQS/SimpleJson.g.cs b/src/NServiceBus.AwsLambda.SQS/SimpleJson.g.cs deleted file mode 100644 index bda2318..0000000 --- a/src/NServiceBus.AwsLambda.SQS/SimpleJson.g.cs +++ /dev/null @@ -1,2150 +0,0 @@ -//----------------------------------------------------------------------- -// -// Copyright (c) 2011, The Outercurve Foundation. -// -// Licensed under the MIT License (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.opensource.org/licenses/mit-license.php -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Nathan Totten (ntotten.com), Jim Zimmerman (jimzimmerman.com) and Prabir Shrestha (prabir.me) -// https://github.com/facebook-csharp-sdk/simple-json -//----------------------------------------------------------------------- - -// VERSION: - -// NOTE: uncomment the following line to make SimpleJson class internal. -#define SIMPLE_JSON_INTERNAL - -// NOTE: uncomment the following line to make JsonArray and JsonObject class internal. -#define SIMPLE_JSON_OBJARRAYINTERNAL - -// NOTE: uncomment the following line to enable dynamic support. -//#define SIMPLE_JSON_DYNAMIC - -// NOTE: uncomment the following line to enable DataContract support. -//#define SIMPLE_JSON_DATACONTRACT - -// NOTE: uncomment the following line to enable IReadOnlyCollection and IReadOnlyList support. -//#define SIMPLE_JSON_READONLY_COLLECTIONS - -// NOTE: uncomment the following line to disable linq expressions/compiled lambda (better performance) instead of method.invoke(). -// define if you are using .net framework <= 3.0 or < WP7.5 -//#define SIMPLE_JSON_NO_LINQ_EXPRESSION - -// NOTE: uncomment the following line if you are compiling under Window Metro style application/library. -// usually already defined in properties -//#define NETFX_CORE; - -// If you are targetting WinStore, WP8 and NET4.5+ PCL make sure to #define SIMPLE_JSON_TYPEINFO; - -// original json parsing code from http://techblog.procurios.nl/k/618/news/view/14605/14863/How-do-I-write-my-own-parser-for-JSON.html - -#if NETFX_CORE -#define SIMPLE_JSON_TYPEINFO -#endif - -using System; -using System.CodeDom.Compiler; -using System.Collections; -using System.Collections.Generic; -#if !SIMPLE_JSON_NO_LINQ_EXPRESSION -using System.Linq.Expressions; -#endif -using System.ComponentModel; -using System.Diagnostics.CodeAnalysis; -#if SIMPLE_JSON_DYNAMIC -using System.Dynamic; -#endif -using System.Globalization; -using System.Reflection; -using System.Runtime.Serialization; -using System.Text; -using SimpleJson.Reflection; - -namespace SimpleJson -{ - /// - /// Represents the json array. - /// - [GeneratedCode("simple-json", "1.0.0")] - [EditorBrowsable(EditorBrowsableState.Never)] - [SuppressMessage("Microsoft.Naming", "CA1710:IdentifiersShouldHaveCorrectSuffix")] -#if SIMPLE_JSON_OBJARRAYINTERNAL - internal -#else - public -#endif - class JsonArray : List - { - /// - /// Initializes a new instance of the class. - /// - public JsonArray() { } - - /// - /// Initializes a new instance of the class. - /// - /// The capacity of the json array. - public JsonArray(int capacity) : base(capacity) { } - - /// - /// The json representation of the array. - /// - /// The json representation of the array. - public override string ToString() - { - return SimpleJson.SerializeObject(this) ?? string.Empty; - } - } - - /// - /// Represents the json object. - /// - [GeneratedCode("simple-json", "1.0.0")] - [EditorBrowsable(EditorBrowsableState.Never)] - [SuppressMessage("Microsoft.Naming", "CA1710:IdentifiersShouldHaveCorrectSuffix")] -#if SIMPLE_JSON_OBJARRAYINTERNAL - internal -#else - public -#endif - class JsonObject : -#if SIMPLE_JSON_DYNAMIC - DynamicObject, -#endif - IDictionary - { - /// - /// The internal member dictionary. - /// - private readonly Dictionary _members; - - /// - /// Initializes a new instance of . - /// - public JsonObject() - { - _members = new Dictionary(); - } - - /// - /// Initializes a new instance of . - /// - /// The implementation to use when comparing keys, or null to use the default for the type of the key. - public JsonObject(IEqualityComparer comparer) - { - _members = new Dictionary(comparer); - } - - /// - /// Gets the at the specified index. - /// - /// - public object this[int index] - { - get { return GetAtIndex(_members, index); } - } - - internal static object GetAtIndex(IDictionary obj, int index) - { - if (obj == null) - throw new ArgumentNullException("obj"); - if (index >= obj.Count) - throw new ArgumentOutOfRangeException("index"); - int i = 0; - foreach (KeyValuePair o in obj) - if (i++ == index) return o.Value; - return null; - } - - /// - /// Adds the specified key. - /// - /// The key. - /// The value. - public void Add(string key, object value) - { - _members.Add(key, value); - } - - /// - /// Determines whether the specified key contains key. - /// - /// The key. - /// - /// true if the specified key contains key; otherwise, false. - /// - public bool ContainsKey(string key) - { - return _members.ContainsKey(key); - } - - /// - /// Gets the keys. - /// - /// The keys. - public ICollection Keys - { - get { return _members.Keys; } - } - - /// - /// Removes the specified key. - /// - /// The key. - /// - public bool Remove(string key) - { - return _members.Remove(key); - } - - /// - /// Tries the get value. - /// - /// The key. - /// The value. - /// - public bool TryGetValue(string key, out object value) - { - return _members.TryGetValue(key, out value); - } - - /// - /// Gets the values. - /// - /// The values. - public ICollection Values - { - get { return _members.Values; } - } - - /// - /// Gets or sets the with the specified key. - /// - /// - public object this[string key] - { - get { return _members[key]; } - set { _members[key] = value; } - } - - /// - /// Adds the specified item. - /// - /// The item. - public void Add(KeyValuePair item) - { - _members.Add(item.Key, item.Value); - } - - /// - /// Clears this instance. - /// - public void Clear() - { - _members.Clear(); - } - - /// - /// Determines whether [contains] [the specified item]. - /// - /// The item. - /// - /// true if [contains] [the specified item]; otherwise, false. - /// - public bool Contains(KeyValuePair item) - { - return _members.ContainsKey(item.Key) && _members[item.Key] == item.Value; - } - - /// - /// Copies to. - /// - /// The array. - /// Index of the array. - public void CopyTo(KeyValuePair[] array, int arrayIndex) - { - if (array == null) throw new ArgumentNullException("array"); - int num = Count; - foreach (KeyValuePair kvp in this) - { - array[arrayIndex++] = kvp; - if (--num <= 0) - return; - } - } - - /// - /// Gets the count. - /// - /// The count. - public int Count - { - get { return _members.Count; } - } - - /// - /// Gets a value indicating whether this instance is read only. - /// - /// - /// true if this instance is read only; otherwise, false. - /// - public bool IsReadOnly - { - get { return false; } - } - - /// - /// Removes the specified item. - /// - /// The item. - /// - public bool Remove(KeyValuePair item) - { - return _members.Remove(item.Key); - } - - /// - /// Gets the enumerator. - /// - /// - public IEnumerator> GetEnumerator() - { - return _members.GetEnumerator(); - } - - /// - /// Returns an enumerator that iterates through a collection. - /// - /// - /// An object that can be used to iterate through the collection. - /// - IEnumerator IEnumerable.GetEnumerator() - { - return _members.GetEnumerator(); - } - - /// - /// Returns a json that represents the current . - /// - /// - /// A json that represents the current . - /// - public override string ToString() - { - return SimpleJson.SerializeObject(this); - } - -#if SIMPLE_JSON_DYNAMIC - /// - /// Provides implementation for type conversion operations. Classes derived from the class can override this method to specify dynamic behavior for operations that convert an object from one type to another. - /// - /// Provides information about the conversion operation. The binder.Type property provides the type to which the object must be converted. For example, for the statement (String)sampleObject in C# (CType(sampleObject, Type) in Visual Basic), where sampleObject is an instance of the class derived from the class, binder.Type returns the type. The binder.Explicit property provides information about the kind of conversion that occurs. It returns true for explicit conversion and false for implicit conversion. - /// The result of the type conversion operation. - /// - /// Alwasy returns true. - /// - public override bool TryConvert(ConvertBinder binder, out object result) - { - // - if (binder == null) - throw new ArgumentNullException("binder"); - // - Type targetType = binder.Type; - - if ((targetType == typeof(IEnumerable)) || - (targetType == typeof(IEnumerable>)) || - (targetType == typeof(IDictionary)) || - (targetType == typeof(IDictionary))) - { - result = this; - return true; - } - - return base.TryConvert(binder, out result); - } - - /// - /// Provides the implementation for operations that delete an object member. This method is not intended for use in C# or Visual Basic. - /// - /// Provides information about the deletion. - /// - /// Alwasy returns true. - /// - public override bool TryDeleteMember(DeleteMemberBinder binder) - { - // - if (binder == null) - throw new ArgumentNullException("binder"); - // - return _members.Remove(binder.Name); - } - - /// - /// Provides the implementation for operations that get a value by index. Classes derived from the class can override this method to specify dynamic behavior for indexing operations. - /// - /// Provides information about the operation. - /// The indexes that are used in the operation. For example, for the sampleObject[3] operation in C# (sampleObject(3) in Visual Basic), where sampleObject is derived from the DynamicObject class, is equal to 3. - /// The result of the index operation. - /// - /// Alwasy returns true. - /// - public override bool TryGetIndex(GetIndexBinder binder, object[] indexes, out object result) - { - if (indexes == null) throw new ArgumentNullException("indexes"); - if (indexes.Length == 1) - { - result = ((IDictionary)this)[(string)indexes[0]]; - return true; - } - result = null; - return true; - } - - /// - /// Provides the implementation for operations that get member values. Classes derived from the class can override this method to specify dynamic behavior for operations such as getting a value for a property. - /// - /// Provides information about the object that called the dynamic operation. The binder.Name property provides the name of the member on which the dynamic operation is performed. For example, for the Console.WriteLine(sampleObject.SampleProperty) statement, where sampleObject is an instance of the class derived from the class, binder.Name returns "SampleProperty". The binder.IgnoreCase property specifies whether the member name is case-sensitive. - /// The result of the get operation. For example, if the method is called for a property, you can assign the property value to . - /// - /// Alwasy returns true. - /// - public override bool TryGetMember(GetMemberBinder binder, out object result) - { - object value; - if (_members.TryGetValue(binder.Name, out value)) - { - result = value; - return true; - } - result = null; - return true; - } - - /// - /// Provides the implementation for operations that set a value by index. Classes derived from the class can override this method to specify dynamic behavior for operations that access objects by a specified index. - /// - /// Provides information about the operation. - /// The indexes that are used in the operation. For example, for the sampleObject[3] = 10 operation in C# (sampleObject(3) = 10 in Visual Basic), where sampleObject is derived from the class, is equal to 3. - /// The value to set to the object that has the specified index. For example, for the sampleObject[3] = 10 operation in C# (sampleObject(3) = 10 in Visual Basic), where sampleObject is derived from the class, is equal to 10. - /// - /// true if the operation is successful; otherwise, false. If this method returns false, the run-time binder of the language determines the behavior. (In most cases, a language-specific run-time exception is thrown. - /// - public override bool TrySetIndex(SetIndexBinder binder, object[] indexes, object value) - { - if (indexes == null) throw new ArgumentNullException("indexes"); - if (indexes.Length == 1) - { - ((IDictionary)this)[(string)indexes[0]] = value; - return true; - } - return base.TrySetIndex(binder, indexes, value); - } - - /// - /// Provides the implementation for operations that set member values. Classes derived from the class can override this method to specify dynamic behavior for operations such as setting a value for a property. - /// - /// Provides information about the object that called the dynamic operation. The binder.Name property provides the name of the member to which the value is being assigned. For example, for the statement sampleObject.SampleProperty = "Test", where sampleObject is an instance of the class derived from the class, binder.Name returns "SampleProperty". The binder.IgnoreCase property specifies whether the member name is case-sensitive. - /// The value to set to the member. For example, for sampleObject.SampleProperty = "Test", where sampleObject is an instance of the class derived from the class, the is "Test". - /// - /// true if the operation is successful; otherwise, false. If this method returns false, the run-time binder of the language determines the behavior. (In most cases, a language-specific run-time exception is thrown.) - /// - public override bool TrySetMember(SetMemberBinder binder, object value) - { - // - if (binder == null) - throw new ArgumentNullException("binder"); - // - _members[binder.Name] = value; - return true; - } - - /// - /// Returns the enumeration of all dynamic member names. - /// - /// - /// A sequence that contains dynamic member names. - /// - public override IEnumerable GetDynamicMemberNames() - { - foreach (var key in Keys) - yield return key; - } -#endif - } -} - -namespace SimpleJson -{ - /// - /// This class encodes and decodes JSON strings. - /// Spec. details, see http://www.json.org/ - /// - /// JSON uses Arrays and Objects. These correspond here to the datatypes JsonArray(IList<object>) and JsonObject(IDictionary<string,object>). - /// All numbers are parsed to doubles. - /// - [GeneratedCode("simple-json", "1.0.0")] -#if SIMPLE_JSON_INTERNAL - internal -#else - public -#endif - static class SimpleJson - { - private const int TOKEN_NONE = 0; - private const int TOKEN_CURLY_OPEN = 1; - private const int TOKEN_CURLY_CLOSE = 2; - private const int TOKEN_SQUARED_OPEN = 3; - private const int TOKEN_SQUARED_CLOSE = 4; - private const int TOKEN_COLON = 5; - private const int TOKEN_COMMA = 6; - private const int TOKEN_STRING = 7; - private const int TOKEN_NUMBER = 8; - private const int TOKEN_TRUE = 9; - private const int TOKEN_FALSE = 10; - private const int TOKEN_NULL = 11; - private const int BUILDER_CAPACITY = 2000; - - private static readonly char[] EscapeTable; - private static readonly char[] EscapeCharacters = new char[] { '"', '\\', '\b', '\f', '\n', '\r', '\t' }; - private static readonly string EscapeCharactersString = new string(EscapeCharacters); - - static SimpleJson() - { - EscapeTable = new char[93]; - EscapeTable['"'] = '"'; - EscapeTable['\\'] = '\\'; - EscapeTable['\b'] = 'b'; - EscapeTable['\f'] = 'f'; - EscapeTable['\n'] = 'n'; - EscapeTable['\r'] = 'r'; - EscapeTable['\t'] = 't'; - } - - /// - /// Parses the string json into a value - /// - /// A JSON string. - /// An IList<object>, a IDictionary<string,object>, a double, a string, null, true, or false - public static object DeserializeObject(string json) - { - object obj; - if (TryDeserializeObject(json, out obj)) - return obj; - throw new SerializationException("Invalid JSON string"); - } - - /// - /// Try parsing the json string into a value. - /// - /// - /// A JSON string. - /// - /// - /// The object. - /// - /// - /// Returns true if successful otherwise false. - /// - [SuppressMessage("Microsoft.Design", "CA1007:UseGenericsWhereAppropriate", Justification="Need to support .NET 2")] - public static bool TryDeserializeObject(string json, out object obj) - { - bool success = true; - if (json != null) - { - char[] charArray = json.ToCharArray(); - int index = 0; - obj = ParseValue(charArray, ref index, ref success); - } - else - obj = null; - - return success; - } - - public static object DeserializeObject(string json, Type type, IJsonSerializerStrategy jsonSerializerStrategy) - { - object jsonObject = DeserializeObject(json); - return type == null || jsonObject != null && ReflectionUtils.IsAssignableFrom(jsonObject.GetType(), type) - ? jsonObject - : (jsonSerializerStrategy ?? CurrentJsonSerializerStrategy).DeserializeObject(jsonObject, type); - } - - public static object DeserializeObject(string json, Type type) - { - return DeserializeObject(json, type, null); - } - - public static T DeserializeObject(string json, IJsonSerializerStrategy jsonSerializerStrategy) - { - return (T)DeserializeObject(json, typeof(T), jsonSerializerStrategy); - } - - public static T DeserializeObject(string json) - { - return (T)DeserializeObject(json, typeof(T), null); - } - - /// - /// Converts a IDictionary<string,object> / IList<object> object into a JSON string - /// - /// A IDictionary<string,object> / IList<object> - /// Serializer strategy to use - /// A JSON encoded string, or null if object 'json' is not serializable - public static string SerializeObject(object json, IJsonSerializerStrategy jsonSerializerStrategy) - { - StringBuilder builder = new StringBuilder(BUILDER_CAPACITY); - bool success = SerializeValue(jsonSerializerStrategy, json, builder); - return (success ? builder.ToString() : null); - } - - public static string SerializeObject(object json) - { - return SerializeObject(json, CurrentJsonSerializerStrategy); - } - - public static string EscapeToJavascriptString(string jsonString) - { - if (string.IsNullOrEmpty(jsonString)) - return jsonString; - - StringBuilder sb = new StringBuilder(); - char c; - - for (int i = 0; i < jsonString.Length; ) - { - c = jsonString[i++]; - - if (c == '\\') - { - int remainingLength = jsonString.Length - i; - if (remainingLength >= 2) - { - char lookahead = jsonString[i]; - if (lookahead == '\\') - { - sb.Append('\\'); - ++i; - } - else if (lookahead == '"') - { - sb.Append("\""); - ++i; - } - else if (lookahead == 't') - { - sb.Append('\t'); - ++i; - } - else if (lookahead == 'b') - { - sb.Append('\b'); - ++i; - } - else if (lookahead == 'n') - { - sb.Append('\n'); - ++i; - } - else if (lookahead == 'r') - { - sb.Append('\r'); - ++i; - } - } - } - else - { - sb.Append(c); - } - } - return sb.ToString(); - } - - static IDictionary ParseObject(char[] json, ref int index, ref bool success) - { - IDictionary table = new JsonObject(); - int token; - - // { - NextToken(json, ref index); - - bool done = false; - while (!done) - { - token = LookAhead(json, index); - if (token == TOKEN_NONE) - { - success = false; - return null; - } - else if (token == TOKEN_COMMA) - NextToken(json, ref index); - else if (token == TOKEN_CURLY_CLOSE) - { - NextToken(json, ref index); - return table; - } - else - { - // name - string name = ParseString(json, ref index, ref success); - if (!success) - { - success = false; - return null; - } - // : - token = NextToken(json, ref index); - if (token != TOKEN_COLON) - { - success = false; - return null; - } - // value - object value = ParseValue(json, ref index, ref success); - if (!success) - { - success = false; - return null; - } - table[name] = value; - } - } - return table; - } - - static JsonArray ParseArray(char[] json, ref int index, ref bool success) - { - JsonArray array = new JsonArray(); - - // [ - NextToken(json, ref index); - - bool done = false; - while (!done) - { - int token = LookAhead(json, index); - if (token == TOKEN_NONE) - { - success = false; - return null; - } - else if (token == TOKEN_COMMA) - NextToken(json, ref index); - else if (token == TOKEN_SQUARED_CLOSE) - { - NextToken(json, ref index); - break; - } - else - { - object value = ParseValue(json, ref index, ref success); - if (!success) - return null; - array.Add(value); - } - } - return array; - } - - static object ParseValue(char[] json, ref int index, ref bool success) - { - switch (LookAhead(json, index)) - { - case TOKEN_STRING: - return ParseString(json, ref index, ref success); - case TOKEN_NUMBER: - return ParseNumber(json, ref index, ref success); - case TOKEN_CURLY_OPEN: - return ParseObject(json, ref index, ref success); - case TOKEN_SQUARED_OPEN: - return ParseArray(json, ref index, ref success); - case TOKEN_TRUE: - NextToken(json, ref index); - return true; - case TOKEN_FALSE: - NextToken(json, ref index); - return false; - case TOKEN_NULL: - NextToken(json, ref index); - return null; - case TOKEN_NONE: - break; - } - success = false; - return null; - } - - static string ParseString(char[] json, ref int index, ref bool success) - { - StringBuilder s = new StringBuilder(BUILDER_CAPACITY); - char c; - - EatWhitespace(json, ref index); - - // " - c = json[index++]; - bool complete = false; - while (!complete) - { - if (index == json.Length) - break; - - c = json[index++]; - if (c == '"') - { - complete = true; - break; - } - else if (c == '\\') - { - if (index == json.Length) - break; - c = json[index++]; - if (c == '"') - s.Append('"'); - else if (c == '\\') - s.Append('\\'); - else if (c == '/') - s.Append('/'); - else if (c == 'b') - s.Append('\b'); - else if (c == 'f') - s.Append('\f'); - else if (c == 'n') - s.Append('\n'); - else if (c == 'r') - s.Append('\r'); - else if (c == 't') - s.Append('\t'); - else if (c == 'u') - { - int remainingLength = json.Length - index; - if (remainingLength >= 4) - { - // parse the 32 bit hex into an integer codepoint - uint codePoint; - if (!(success = UInt32.TryParse(new string(json, index, 4), NumberStyles.HexNumber, CultureInfo.InvariantCulture, out codePoint))) - return ""; - - // convert the integer codepoint to a unicode char and add to string - if (0xD800 <= codePoint && codePoint <= 0xDBFF) // if high surrogate - { - index += 4; // skip 4 chars - remainingLength = json.Length - index; - if (remainingLength >= 6) - { - uint lowCodePoint; - if (new string(json, index, 2) == "\\u" && UInt32.TryParse(new string(json, index + 2, 4), NumberStyles.HexNumber, CultureInfo.InvariantCulture, out lowCodePoint)) - { - if (0xDC00 <= lowCodePoint && lowCodePoint <= 0xDFFF) // if low surrogate - { - s.Append((char)codePoint); - s.Append((char)lowCodePoint); - index += 6; // skip 6 chars - continue; - } - } - } - success = false; // invalid surrogate pair - return ""; - } - s.Append(ConvertFromUtf32((int)codePoint)); - // skip 4 chars - index += 4; - } - else - break; - } - } - else - s.Append(c); - } - if (!complete) - { - success = false; - return null; - } - return s.ToString(); - } - - private static string ConvertFromUtf32(int utf32) - { - // http://www.java2s.com/Open-Source/CSharp/2.6.4-mono-.net-core/System/System/Char.cs.htm - if (utf32 < 0 || utf32 > 0x10FFFF) - throw new ArgumentOutOfRangeException("utf32", "The argument must be from 0 to 0x10FFFF."); - if (0xD800 <= utf32 && utf32 <= 0xDFFF) - throw new ArgumentOutOfRangeException("utf32", "The argument must not be in surrogate pair range."); - if (utf32 < 0x10000) - return new string((char)utf32, 1); - utf32 -= 0x10000; - return new string(new char[] { (char)((utf32 >> 10) + 0xD800), (char)(utf32 % 0x0400 + 0xDC00) }); - } - - static object ParseNumber(char[] json, ref int index, ref bool success) - { - EatWhitespace(json, ref index); - int lastIndex = GetLastIndexOfNumber(json, index); - int charLength = (lastIndex - index) + 1; - object returnNumber; - string str = new string(json, index, charLength); - if (str.IndexOf(".", StringComparison.OrdinalIgnoreCase) != -1 || str.IndexOf("e", StringComparison.OrdinalIgnoreCase) != -1) - { - double number; - success = double.TryParse(new string(json, index, charLength), NumberStyles.Any, CultureInfo.InvariantCulture, out number); - returnNumber = number; - } - else - { - long number; - success = long.TryParse(new string(json, index, charLength), NumberStyles.Any, CultureInfo.InvariantCulture, out number); - returnNumber = number; - } - index = lastIndex + 1; - return returnNumber; - } - - static int GetLastIndexOfNumber(char[] json, int index) - { - int lastIndex; - for (lastIndex = index; lastIndex < json.Length; lastIndex++) - if ("0123456789+-.eE".IndexOf(json[lastIndex]) == -1) break; - return lastIndex - 1; - } - - static void EatWhitespace(char[] json, ref int index) - { - for (; index < json.Length; index++) - if (" \t\n\r\b\f".IndexOf(json[index]) == -1) break; - } - - static int LookAhead(char[] json, int index) - { - int saveIndex = index; - return NextToken(json, ref saveIndex); - } - - [SuppressMessage("Microsoft.Maintainability", "CA1502:AvoidExcessiveComplexity")] - static int NextToken(char[] json, ref int index) - { - EatWhitespace(json, ref index); - if (index == json.Length) - return TOKEN_NONE; - char c = json[index]; - index++; - switch (c) - { - case '{': - return TOKEN_CURLY_OPEN; - case '}': - return TOKEN_CURLY_CLOSE; - case '[': - return TOKEN_SQUARED_OPEN; - case ']': - return TOKEN_SQUARED_CLOSE; - case ',': - return TOKEN_COMMA; - case '"': - return TOKEN_STRING; - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': - case '-': - return TOKEN_NUMBER; - case ':': - return TOKEN_COLON; - } - index--; - int remainingLength = json.Length - index; - // false - if (remainingLength >= 5) - { - if (json[index] == 'f' && json[index + 1] == 'a' && json[index + 2] == 'l' && json[index + 3] == 's' && json[index + 4] == 'e') - { - index += 5; - return TOKEN_FALSE; - } - } - // true - if (remainingLength >= 4) - { - if (json[index] == 't' && json[index + 1] == 'r' && json[index + 2] == 'u' && json[index + 3] == 'e') - { - index += 4; - return TOKEN_TRUE; - } - } - // null - if (remainingLength >= 4) - { - if (json[index] == 'n' && json[index + 1] == 'u' && json[index + 2] == 'l' && json[index + 3] == 'l') - { - index += 4; - return TOKEN_NULL; - } - } - return TOKEN_NONE; - } - - static bool SerializeValue(IJsonSerializerStrategy jsonSerializerStrategy, object value, StringBuilder builder) - { - bool success = true; - string stringValue = value as string; - if (stringValue != null) - success = SerializeString(stringValue, builder); - else - { - IDictionary dict = value as IDictionary; - if (dict != null) - { - success = SerializeObject(jsonSerializerStrategy, dict.Keys, dict.Values, builder); - } - else - { - IDictionary stringDictionary = value as IDictionary; - if (stringDictionary != null) - { - success = SerializeObject(jsonSerializerStrategy, stringDictionary.Keys, stringDictionary.Values, builder); - } - else - { - IEnumerable enumerableValue = value as IEnumerable; - if (enumerableValue != null) - success = SerializeArray(jsonSerializerStrategy, enumerableValue, builder); - else if (IsNumeric(value)) - success = SerializeNumber(value, builder); - else if (value is bool) - builder.Append((bool)value ? "true" : "false"); - else if (value == null) - builder.Append("null"); - else - { - object serializedObject; - success = jsonSerializerStrategy.TrySerializeNonPrimitiveObject(value, out serializedObject); - if (success) - SerializeValue(jsonSerializerStrategy, serializedObject, builder); - } - } - } - } - return success; - } - - static bool SerializeObject(IJsonSerializerStrategy jsonSerializerStrategy, IEnumerable keys, IEnumerable values, StringBuilder builder) - { - builder.Append("{"); - IEnumerator ke = keys.GetEnumerator(); - IEnumerator ve = values.GetEnumerator(); - bool first = true; - while (ke.MoveNext() && ve.MoveNext()) - { - object key = ke.Current; - object value = ve.Current; - if (!first) - builder.Append(","); - string stringKey = key as string; - if (stringKey != null) - SerializeString(stringKey, builder); - else - if (!SerializeValue(jsonSerializerStrategy, value, builder)) return false; - builder.Append(":"); - if (!SerializeValue(jsonSerializerStrategy, value, builder)) - return false; - first = false; - } - builder.Append("}"); - return true; - } - - static bool SerializeArray(IJsonSerializerStrategy jsonSerializerStrategy, IEnumerable anArray, StringBuilder builder) - { - builder.Append("["); - bool first = true; - foreach (object value in anArray) - { - if (!first) - builder.Append(","); - if (!SerializeValue(jsonSerializerStrategy, value, builder)) - return false; - first = false; - } - builder.Append("]"); - return true; - } - - static bool SerializeString(string aString, StringBuilder builder) - { - // Happy path if there's nothing to be escaped. IndexOfAny is highly optimized (and unmanaged) - if (aString.IndexOfAny(EscapeCharacters) == -1) - { - builder.Append('"'); - builder.Append(aString); - builder.Append('"'); - - return true; - } - - builder.Append('"'); - int safeCharacterCount = 0; - char[] charArray = aString.ToCharArray(); - - for (int i = 0; i < charArray.Length; i++) - { - char c = charArray[i]; - - // Non ascii characters are fine, buffer them up and send them to the builder - // in larger chunks if possible. The escape table is a 1:1 translation table - // with \0 [default(char)] denoting a safe character. - if (c >= EscapeTable.Length || EscapeTable[c] == default(char)) - { - safeCharacterCount++; - } - else - { - if (safeCharacterCount > 0) - { - builder.Append(charArray, i - safeCharacterCount, safeCharacterCount); - safeCharacterCount = 0; - } - - builder.Append('\\'); - builder.Append(EscapeTable[c]); - } - } - - if (safeCharacterCount > 0) - { - builder.Append(charArray, charArray.Length - safeCharacterCount, safeCharacterCount); - } - - builder.Append('"'); - return true; - } - - static bool SerializeNumber(object number, StringBuilder builder) - { - if (number is long) - builder.Append(((long)number).ToString(CultureInfo.InvariantCulture)); - else if (number is ulong) - builder.Append(((ulong)number).ToString(CultureInfo.InvariantCulture)); - else if (number is int) - builder.Append(((int)number).ToString(CultureInfo.InvariantCulture)); - else if (number is uint) - builder.Append(((uint)number).ToString(CultureInfo.InvariantCulture)); - else if (number is decimal) - builder.Append(((decimal)number).ToString(CultureInfo.InvariantCulture)); - else if (number is float) - builder.Append(((float)number).ToString(CultureInfo.InvariantCulture)); - else - builder.Append(Convert.ToDouble(number, CultureInfo.InvariantCulture).ToString("r", CultureInfo.InvariantCulture)); - return true; - } - - /// - /// Determines if a given object is numeric in any way - /// (can be integer, double, null, etc). - /// - static bool IsNumeric(object value) - { - if (value is sbyte) return true; - if (value is byte) return true; - if (value is short) return true; - if (value is ushort) return true; - if (value is int) return true; - if (value is uint) return true; - if (value is long) return true; - if (value is ulong) return true; - if (value is float) return true; - if (value is double) return true; - if (value is decimal) return true; - return false; - } - - private static IJsonSerializerStrategy _currentJsonSerializerStrategy; - public static IJsonSerializerStrategy CurrentJsonSerializerStrategy - { - get - { - return _currentJsonSerializerStrategy ?? - (_currentJsonSerializerStrategy = -#if SIMPLE_JSON_DATACONTRACT - DataContractJsonSerializerStrategy -#else - PocoJsonSerializerStrategy -#endif -); - } - set - { - _currentJsonSerializerStrategy = value; - } - } - - private static PocoJsonSerializerStrategy _pocoJsonSerializerStrategy; - [EditorBrowsable(EditorBrowsableState.Advanced)] - public static PocoJsonSerializerStrategy PocoJsonSerializerStrategy - { - get - { - return _pocoJsonSerializerStrategy ?? (_pocoJsonSerializerStrategy = new PocoJsonSerializerStrategy()); - } - } - -#if SIMPLE_JSON_DATACONTRACT - - private static DataContractJsonSerializerStrategy _dataContractJsonSerializerStrategy; - [System.ComponentModel.EditorBrowsable(EditorBrowsableState.Advanced)] - public static DataContractJsonSerializerStrategy DataContractJsonSerializerStrategy - { - get - { - return _dataContractJsonSerializerStrategy ?? (_dataContractJsonSerializerStrategy = new DataContractJsonSerializerStrategy()); - } - } - -#endif - } - - [GeneratedCode("simple-json", "1.0.0")] -#if SIMPLE_JSON_INTERNAL - internal -#else - public -#endif - interface IJsonSerializerStrategy - { - [SuppressMessage("Microsoft.Design", "CA1007:UseGenericsWhereAppropriate", Justification="Need to support .NET 2")] - bool TrySerializeNonPrimitiveObject(object input, out object output); - object DeserializeObject(object value, Type type); - } - - [GeneratedCode("simple-json", "1.0.0")] -#if SIMPLE_JSON_INTERNAL - internal -#else - public -#endif - class PocoJsonSerializerStrategy : IJsonSerializerStrategy - { - internal IDictionary ConstructorCache; - internal IDictionary> GetCache; - internal IDictionary>> SetCache; - - internal static readonly Type[] EmptyTypes = new Type[0]; - internal static readonly Type[] ArrayConstructorParameterTypes = new Type[] { typeof(int) }; - - private static readonly string[] Iso8601Format = new string[] - { - @"yyyy-MM-dd\THH:mm:ss.FFFFFFF\Z", - @"yyyy-MM-dd\THH:mm:ss\Z", - @"yyyy-MM-dd\THH:mm:ssK" - }; - - public PocoJsonSerializerStrategy() - { - ConstructorCache = new ReflectionUtils.ThreadSafeDictionary(ConstructorDelegateFactory); - GetCache = new ReflectionUtils.ThreadSafeDictionary>(GetterValueFactory); - SetCache = new ReflectionUtils.ThreadSafeDictionary>>(SetterValueFactory); - } - - protected virtual string MapClrMemberNameToJsonFieldName(string clrPropertyName) - { - return clrPropertyName; - } - - internal virtual ReflectionUtils.ConstructorDelegate ConstructorDelegateFactory(Type key) - { - // We need List(int) constructor so that DeserializeObject method will work for generating IList-declared values - var needsCapacityArgument = key.IsArray || key.IsConstructedGenericType && key.GetGenericTypeDefinition() == typeof(List<>); - return ReflectionUtils.GetConstructor(key, needsCapacityArgument ? ArrayConstructorParameterTypes : EmptyTypes); - } - - internal virtual IDictionary GetterValueFactory(Type type) - { - IDictionary result = new Dictionary(); - foreach (PropertyInfo propertyInfo in ReflectionUtils.GetProperties(type)) - { - if (propertyInfo.CanRead) - { - MethodInfo getMethod = ReflectionUtils.GetGetterMethodInfo(propertyInfo); - if (getMethod.IsStatic || !getMethod.IsPublic) - continue; - result[MapClrMemberNameToJsonFieldName(propertyInfo.Name)] = ReflectionUtils.GetGetMethod(propertyInfo); - } - } - foreach (FieldInfo fieldInfo in ReflectionUtils.GetFields(type)) - { - if (fieldInfo.IsStatic || !fieldInfo.IsPublic) - continue; - result[MapClrMemberNameToJsonFieldName(fieldInfo.Name)] = ReflectionUtils.GetGetMethod(fieldInfo); - } - return result; - } - - internal virtual IDictionary> SetterValueFactory(Type type) - { - IDictionary> result = new Dictionary>(); - foreach (PropertyInfo propertyInfo in ReflectionUtils.GetProperties(type)) - { - if (propertyInfo.CanWrite) - { - MethodInfo setMethod = ReflectionUtils.GetSetterMethodInfo(propertyInfo); - if (setMethod.IsStatic || !setMethod.IsPublic) - continue; - result[MapClrMemberNameToJsonFieldName(propertyInfo.Name)] = new KeyValuePair(propertyInfo.PropertyType, ReflectionUtils.GetSetMethod(propertyInfo)); - } - } - foreach (FieldInfo fieldInfo in ReflectionUtils.GetFields(type)) - { - if (fieldInfo.IsInitOnly || fieldInfo.IsStatic || !fieldInfo.IsPublic) - continue; - result[MapClrMemberNameToJsonFieldName(fieldInfo.Name)] = new KeyValuePair(fieldInfo.FieldType, ReflectionUtils.GetSetMethod(fieldInfo)); - } - return result; - } - - public virtual bool TrySerializeNonPrimitiveObject(object input, out object output) - { - return TrySerializeKnownTypes(input, out output) || TrySerializeUnknownTypes(input, out output); - } - - [SuppressMessage("Microsoft.Maintainability", "CA1502:AvoidExcessiveComplexity")] - public virtual object DeserializeObject(object value, Type type) - { - if (type == null) throw new ArgumentNullException("type"); - string str = value as string; - - if (type == typeof (Guid) && string.IsNullOrEmpty(str)) - return default(Guid); - - if (type.IsEnum) - { - type = type.GetEnumUnderlyingType(); - } - - if (value == null) - return null; - - object obj = null; - - if (str != null) - { - if (str.Length != 0) // We know it can't be null now. - { - if (type == typeof(TimeSpan) || (ReflectionUtils.IsNullableType(type) && Nullable.GetUnderlyingType(type) == typeof(TimeSpan))) - return TimeSpan.ParseExact(str, "c", CultureInfo.InvariantCulture); - if (type == typeof(DateTime) || (ReflectionUtils.IsNullableType(type) && Nullable.GetUnderlyingType(type) == typeof(DateTime))) - return DateTime.TryParseExact(str, Iso8601Format, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var result) - ? result : DateTime.Parse(str, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind); - if (type == typeof(DateTimeOffset) || (ReflectionUtils.IsNullableType(type) && Nullable.GetUnderlyingType(type) == typeof(DateTimeOffset))) - return DateTimeOffset.TryParseExact(str, Iso8601Format, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var result) - ? result : DateTimeOffset.Parse(str, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind); - if (type == typeof(Guid) || (ReflectionUtils.IsNullableType(type) && Nullable.GetUnderlyingType(type) == typeof(Guid))) - return new Guid(str); - if (type == typeof(Uri)) - { - bool isValid = Uri.IsWellFormedUriString(str, UriKind.RelativeOrAbsolute); - - Uri result; - if (isValid && Uri.TryCreate(str, UriKind.RelativeOrAbsolute, out result)) - return result; - - return null; - } - - if (type == typeof(string)) - return str; - - return Convert.ChangeType(str, type, CultureInfo.InvariantCulture); - } - else - { - if (type == typeof(Guid)) - obj = default(Guid); - else if (ReflectionUtils.IsNullableType(type) && Nullable.GetUnderlyingType(type) == typeof(Guid)) - obj = null; - else - obj = str; - } - // Empty string case - if (!ReflectionUtils.IsNullableType(type) && Nullable.GetUnderlyingType(type) == typeof(Guid)) - return str; - } - else if (value is bool) - return value; - - bool valueIsLong = value is long; - bool valueIsDouble = value is double; - if ((valueIsLong && type == typeof(long)) || (valueIsDouble && type == typeof(double))) - return value; - if ((valueIsDouble && type != typeof(double)) || (valueIsLong && type != typeof(long))) - { - obj = type == typeof(int) || type == typeof(long) || type == typeof(double) || type == typeof(float) || type == typeof(bool) || type == typeof(decimal) || type == typeof(byte) || type == typeof(short) - ? Convert.ChangeType(value, type, CultureInfo.InvariantCulture) - : value; - } - else - { - IDictionary objects = value as IDictionary; - if (objects != null) - { - IDictionary jsonObject = objects; - - if (ReflectionUtils.IsTypeDictionary(type)) - { - // if dictionary then - Type[] types = ReflectionUtils.GetGenericTypeArguments(type); - Type keyType = types[0]; - Type valueType = types[1]; - - Type genericType = typeof(Dictionary<,>).MakeGenericType(keyType, valueType); - - IDictionary dict = (IDictionary)ConstructorCache[genericType](); - - foreach (KeyValuePair kvp in jsonObject) - dict.Add(kvp.Key, DeserializeObject(kvp.Value, valueType)); - - obj = dict; - } - else - { - if (type == typeof(object)) - obj = value; - else - { - obj = ConstructorCache[type](); - foreach (KeyValuePair> setter in SetCache[type]) - { - object jsonValue; - if (jsonObject.TryGetValue(setter.Key, out jsonValue)) - { - jsonValue = DeserializeObject(jsonValue, setter.Value.Key); - setter.Value.Value(obj, jsonValue); - } - } - } - } - } - else - { - IList valueAsList = value as IList; - if (valueAsList != null) - { - IList jsonObject = valueAsList; - IList list = null; - - if (type.IsArray) - { - list = (IList)ConstructorCache[type](jsonObject.Count); - int i = 0; - foreach (object o in jsonObject) - list[i++] = DeserializeObject(o, type.GetElementType()); - } - else if (ReflectionUtils.IsTypeGenericCollectionInterface(type) || ReflectionUtils.IsAssignableFrom(typeof(IList), type)) - { - Type innerType = ReflectionUtils.GetGenericListElementType(type); - list = (IList)(ConstructorCache[type] ?? ConstructorCache[typeof(List<>).MakeGenericType(innerType)])(jsonObject.Count); - foreach (object o in jsonObject) - list.Add(DeserializeObject(o, innerType)); - } - obj = list; - } - } - return obj; - } - if (ReflectionUtils.IsNullableType(type)) - return ReflectionUtils.ToNullableType(obj, type); - return obj; - } - - protected virtual object SerializeEnum(Enum p) - { - return Convert.ToDouble(p, CultureInfo.InvariantCulture); - } - - [SuppressMessage("Microsoft.Design", "CA1007:UseGenericsWhereAppropriate", Justification="Need to support .NET 2")] - protected virtual bool TrySerializeKnownTypes(object input, out object output) - { - bool returnValue = true; - if (input is DateTime) - output = ((DateTime)input).ToUniversalTime().ToString(Iso8601Format[0], CultureInfo.InvariantCulture); - else if (input is DateTimeOffset) - output = ((DateTimeOffset)input).ToString("o"); - else if (input is Guid) - output = ((Guid)input).ToString("D"); - else if (input is Uri) - output = input.ToString(); - else if (input is TimeSpan) - output = ((TimeSpan)input).ToString("c"); - else - { - Enum inputEnum = input as Enum; - if (inputEnum != null) - output = SerializeEnum(inputEnum); - else - { - returnValue = false; - output = null; - } - } - return returnValue; - } - [SuppressMessage("Microsoft.Design", "CA1007:UseGenericsWhereAppropriate", Justification="Need to support .NET 2")] - protected virtual bool TrySerializeUnknownTypes(object input, out object output) - { - if (input == null) throw new ArgumentNullException("input"); - output = null; - Type type = input.GetType(); - if (type.FullName == null) - return false; - IDictionary obj = new JsonObject(); - IDictionary getters = GetCache[type]; - foreach (KeyValuePair getter in getters) - { - if (getter.Value != null) - obj.Add(MapClrMemberNameToJsonFieldName(getter.Key), getter.Value(input)); - } - output = obj; - return true; - } - } - -#if SIMPLE_JSON_DATACONTRACT - [GeneratedCode("simple-json", "1.0.0")] -#if SIMPLE_JSON_INTERNAL - internal -#else - public -#endif - class DataContractJsonSerializerStrategy : PocoJsonSerializerStrategy - { - public DataContractJsonSerializerStrategy() - { - GetCache = new ReflectionUtils.ThreadSafeDictionary>(GetterValueFactory); - SetCache = new ReflectionUtils.ThreadSafeDictionary>>(SetterValueFactory); - } - - internal override IDictionary GetterValueFactory(Type type) - { - bool hasDataContract = ReflectionUtils.GetAttribute(type, typeof(DataContractAttribute)) != null; - if (!hasDataContract) - return base.GetterValueFactory(type); - string jsonKey; - IDictionary result = new Dictionary(); - foreach (PropertyInfo propertyInfo in ReflectionUtils.GetProperties(type)) - { - if (propertyInfo.CanRead) - { - MethodInfo getMethod = ReflectionUtils.GetGetterMethodInfo(propertyInfo); - if (!getMethod.IsStatic && CanAdd(propertyInfo, out jsonKey)) - result[jsonKey] = ReflectionUtils.GetGetMethod(propertyInfo); - } - } - foreach (FieldInfo fieldInfo in ReflectionUtils.GetFields(type)) - { - if (!fieldInfo.IsStatic && CanAdd(fieldInfo, out jsonKey)) - result[jsonKey] = ReflectionUtils.GetGetMethod(fieldInfo); - } - return result; - } - - internal override IDictionary> SetterValueFactory(Type type) - { - bool hasDataContract = ReflectionUtils.GetAttribute(type, typeof(DataContractAttribute)) != null; - if (!hasDataContract) - return base.SetterValueFactory(type); - string jsonKey; - IDictionary> result = new Dictionary>(); - foreach (PropertyInfo propertyInfo in ReflectionUtils.GetProperties(type)) - { - if (propertyInfo.CanWrite) - { - MethodInfo setMethod = ReflectionUtils.GetSetterMethodInfo(propertyInfo); - if (!setMethod.IsStatic && CanAdd(propertyInfo, out jsonKey)) - result[jsonKey] = new KeyValuePair(propertyInfo.PropertyType, ReflectionUtils.GetSetMethod(propertyInfo)); - } - } - foreach (FieldInfo fieldInfo in ReflectionUtils.GetFields(type)) - { - if (!fieldInfo.IsInitOnly && !fieldInfo.IsStatic && CanAdd(fieldInfo, out jsonKey)) - result[jsonKey] = new KeyValuePair(fieldInfo.FieldType, ReflectionUtils.GetSetMethod(fieldInfo)); - } - // todo implement sorting for DATACONTRACT. - return result; - } - - private static bool CanAdd(MemberInfo info, out string jsonKey) - { - jsonKey = null; - if (ReflectionUtils.GetAttribute(info, typeof(IgnoreDataMemberAttribute)) != null) - return false; - DataMemberAttribute dataMemberAttribute = (DataMemberAttribute)ReflectionUtils.GetAttribute(info, typeof(DataMemberAttribute)); - if (dataMemberAttribute == null) - return false; - jsonKey = string.IsNullOrEmpty(dataMemberAttribute.Name) ? info.Name : dataMemberAttribute.Name; - return true; - } - } - -#endif - - namespace Reflection - { - // This class is meant to be copied into other libraries. So we want to exclude it from Code Analysis rules - // that might be in place in the target project. - [GeneratedCode("reflection-utils", "1.0.0")] -#if SIMPLE_JSON_REFLECTION_UTILS_PUBLIC - public -#else - internal -#endif - class ReflectionUtils - { - private static readonly object[] EmptyObjects = new object[] { }; - - public delegate object GetDelegate(object source); - public delegate void SetDelegate(object source, object value); - public delegate object ConstructorDelegate(params object[] args); - - public delegate TValue ThreadSafeDictionaryValueFactory(TKey key); - -#if SIMPLE_JSON_TYPEINFO - public static TypeInfo GetTypeInfo(Type type) - { - return type.GetTypeInfo(); - } -#else - public static Type GetTypeInfo(Type type) - { - return type; - } -#endif - - public static Attribute GetAttribute(MemberInfo info, Type type) - { -#if SIMPLE_JSON_TYPEINFO - if (info == null || type == null || !info.IsDefined(type)) - return null; - return info.GetCustomAttribute(type); -#else - if (info == null || type == null || !Attribute.IsDefined(info, type)) - return null; - return Attribute.GetCustomAttribute(info, type); -#endif - } - - public static Type GetGenericListElementType(Type type) - { - IEnumerable interfaces; -#if SIMPLE_JSON_TYPEINFO - interfaces = type.GetTypeInfo().ImplementedInterfaces; -#else - interfaces = type.GetInterfaces(); -#endif - foreach (Type implementedInterface in interfaces) - { - if (IsTypeGeneric(implementedInterface) && - implementedInterface.GetGenericTypeDefinition() == typeof (IList<>)) - { - return GetGenericTypeArguments(implementedInterface)[0]; - } - } - return GetGenericTypeArguments(type)[0]; - } - - public static Attribute GetAttribute(Type objectType, Type attributeType) - { - -#if SIMPLE_JSON_TYPEINFO - if (objectType == null || attributeType == null || !objectType.GetTypeInfo().IsDefined(attributeType)) - return null; - return objectType.GetTypeInfo().GetCustomAttribute(attributeType); -#else - if (objectType == null || attributeType == null || !Attribute.IsDefined(objectType, attributeType)) - return null; - return Attribute.GetCustomAttribute(objectType, attributeType); -#endif - } - - public static Type[] GetGenericTypeArguments(Type type) - { -#if SIMPLE_JSON_TYPEINFO - return type.GetTypeInfo().GenericTypeArguments; -#else - return type.GetGenericArguments(); -#endif - } - - public static bool IsTypeGeneric(Type type) - { - return GetTypeInfo(type).IsGenericType; - } - - public static bool IsTypeGenericCollectionInterface(Type type) - { - if (!IsTypeGeneric(type)) - return false; - - Type genericDefinition = type.GetGenericTypeDefinition(); - - return (genericDefinition == typeof(IList<>) - || genericDefinition == typeof(ICollection<>) - || genericDefinition == typeof(IEnumerable<>) -#if SIMPLE_JSON_READONLY_COLLECTIONS - || genericDefinition == typeof(IReadOnlyCollection<>) - || genericDefinition == typeof(IReadOnlyList<>) -#endif - ); - } - - public static bool IsAssignableFrom(Type type1, Type type2) - { - return GetTypeInfo(type1).IsAssignableFrom(GetTypeInfo(type2)); - } - - public static bool IsTypeDictionary(Type type) - { -#if SIMPLE_JSON_TYPEINFO - if (typeof(IDictionary<,>).GetTypeInfo().IsAssignableFrom(type.GetTypeInfo())) - return true; -#else - if (typeof(System.Collections.IDictionary).IsAssignableFrom(type)) - return true; -#endif - if (!GetTypeInfo(type).IsGenericType) - return false; - - Type genericDefinition = type.GetGenericTypeDefinition(); - return genericDefinition == typeof(IDictionary<,>); - } - - public static bool IsNullableType(Type type) - { - return GetTypeInfo(type).IsGenericType && type.GetGenericTypeDefinition() == typeof(Nullable<>); - } - - public static object ToNullableType(object obj, Type nullableType) - { - return obj == null ? null : Convert.ChangeType(obj, Nullable.GetUnderlyingType(nullableType), CultureInfo.InvariantCulture); - } - - public static bool IsValueType(Type type) - { - return GetTypeInfo(type).IsValueType; - } - - public static IEnumerable GetConstructors(Type type) - { -#if SIMPLE_JSON_TYPEINFO - return type.GetTypeInfo().DeclaredConstructors; -#else - return type.GetConstructors(); -#endif - } - - public static ConstructorInfo GetConstructorInfo(Type type, params Type[] argsType) - { - IEnumerable constructorInfos = GetConstructors(type); - int i; - bool matches; - foreach (ConstructorInfo constructorInfo in constructorInfos) - { - ParameterInfo[] parameters = constructorInfo.GetParameters(); - if (argsType.Length != parameters.Length) - continue; - - i = 0; - matches = true; - foreach (ParameterInfo parameterInfo in constructorInfo.GetParameters()) - { - if (parameterInfo.ParameterType != argsType[i]) - { - matches = false; - break; - } - } - - if (matches) - return constructorInfo; - } - - return null; - } - - public static IEnumerable GetProperties(Type type) - { -#if SIMPLE_JSON_TYPEINFO - return type.GetRuntimeProperties(); -#else - return type.GetProperties(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Static); -#endif - } - - public static IEnumerable GetFields(Type type) - { -#if SIMPLE_JSON_TYPEINFO - return type.GetRuntimeFields(); -#else - return type.GetFields(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Static); -#endif - } - - public static MethodInfo GetGetterMethodInfo(PropertyInfo propertyInfo) - { -#if SIMPLE_JSON_TYPEINFO - return propertyInfo.GetMethod; -#else - return propertyInfo.GetGetMethod(true); -#endif - } - - public static MethodInfo GetSetterMethodInfo(PropertyInfo propertyInfo) - { -#if SIMPLE_JSON_TYPEINFO - return propertyInfo.SetMethod; -#else - return propertyInfo.GetSetMethod(true); -#endif - } - - public static ConstructorDelegate GetConstructor(ConstructorInfo constructorInfo) - { -#if SIMPLE_JSON_NO_LINQ_EXPRESSION - return GetConstructorByReflection(constructorInfo); -#else - return GetConstructorByExpression(constructorInfo); -#endif - } - - public static ConstructorDelegate GetConstructor(Type type, params Type[] argsType) - { -#if SIMPLE_JSON_NO_LINQ_EXPRESSION - return GetConstructorByReflection(type, argsType); -#else - return GetConstructorByExpression(type, argsType); -#endif - } - - public static ConstructorDelegate GetConstructorByReflection(ConstructorInfo constructorInfo) - { - return delegate(object[] args) { return constructorInfo.Invoke(args); }; - } - - public static ConstructorDelegate GetConstructorByReflection(Type type, params Type[] argsType) - { - ConstructorInfo constructorInfo = GetConstructorInfo(type, argsType); - - if (constructorInfo == null && argsType.Length == 0 && type.IsValueType) - { - // If it's a struct, then parameterless constructors are implicit - // We can always call Activator.CreateInstance in lieu of a zero-arg constructor - return args => Activator.CreateInstance(type); - } - - return constructorInfo == null ? null : GetConstructorByReflection(constructorInfo); - } - -#if !SIMPLE_JSON_NO_LINQ_EXPRESSION - - public static ConstructorDelegate GetConstructorByExpression(ConstructorInfo constructorInfo) - { - ParameterInfo[] paramsInfo = constructorInfo.GetParameters(); - ParameterExpression param = Expression.Parameter(typeof(object[]), "args"); - Expression[] argsExp = new Expression[paramsInfo.Length]; - for (int i = 0; i < paramsInfo.Length; i++) - { - Expression index = Expression.Constant(i); - Type paramType = paramsInfo[i].ParameterType; - Expression paramAccessorExp = Expression.ArrayIndex(param, index); - Expression paramCastExp = Expression.Convert(paramAccessorExp, paramType); - argsExp[i] = paramCastExp; - } - NewExpression newExp = Expression.New(constructorInfo, argsExp); - Expression> lambda = Expression.Lambda>(newExp, param); - Func compiledLambda = lambda.Compile(); - return delegate(object[] args) { return compiledLambda(args); }; - } - - public static ConstructorDelegate GetConstructorByExpression(Type type, params Type[] argsType) - { - ConstructorInfo constructorInfo = GetConstructorInfo(type, argsType); - - if (constructorInfo == null && argsType.Length == 0 && type.IsValueType) - { - // If it's a struct, then parameterless constructors are implicit - // We can always call Activator.CreateInstance in lieu of a zero-arg constructor - return args => Activator.CreateInstance(type); - } - - return constructorInfo == null ? null : GetConstructorByExpression(constructorInfo); - } - -#endif - - public static GetDelegate GetGetMethod(PropertyInfo propertyInfo) - { -#if SIMPLE_JSON_NO_LINQ_EXPRESSION - return GetGetMethodByReflection(propertyInfo); -#else - return GetGetMethodByExpression(propertyInfo); -#endif - } - - public static GetDelegate GetGetMethod(FieldInfo fieldInfo) - { -#if SIMPLE_JSON_NO_LINQ_EXPRESSION - return GetGetMethodByReflection(fieldInfo); -#else - return GetGetMethodByExpression(fieldInfo); -#endif - } - - public static GetDelegate GetGetMethodByReflection(PropertyInfo propertyInfo) - { - MethodInfo methodInfo = GetGetterMethodInfo(propertyInfo); - return delegate(object source) { return methodInfo.Invoke(source, EmptyObjects); }; - } - - public static GetDelegate GetGetMethodByReflection(FieldInfo fieldInfo) - { - return delegate(object source) { return fieldInfo.GetValue(source); }; - } - -#if !SIMPLE_JSON_NO_LINQ_EXPRESSION - - public static GetDelegate GetGetMethodByExpression(PropertyInfo propertyInfo) - { - MethodInfo getMethodInfo = GetGetterMethodInfo(propertyInfo); - ParameterExpression instance = Expression.Parameter(typeof(object), "instance"); - UnaryExpression instanceCast = (!IsValueType(propertyInfo.DeclaringType)) ? Expression.TypeAs(instance, propertyInfo.DeclaringType) : Expression.Convert(instance, propertyInfo.DeclaringType); - Func compiled = Expression.Lambda>(Expression.TypeAs(Expression.Call(instanceCast, getMethodInfo), typeof(object)), instance).Compile(); - return delegate(object source) { return compiled(source); }; - } - - public static GetDelegate GetGetMethodByExpression(FieldInfo fieldInfo) - { - ParameterExpression instance = Expression.Parameter(typeof(object), "instance"); - MemberExpression member = Expression.Field(Expression.Convert(instance, fieldInfo.DeclaringType), fieldInfo); - GetDelegate compiled = Expression.Lambda(Expression.Convert(member, typeof(object)), instance).Compile(); - return delegate(object source) { return compiled(source); }; - } - -#endif - - public static SetDelegate GetSetMethod(PropertyInfo propertyInfo) - { -#if SIMPLE_JSON_NO_LINQ_EXPRESSION - return GetSetMethodByReflection(propertyInfo); -#else - return GetSetMethodByExpression(propertyInfo); -#endif - } - - public static SetDelegate GetSetMethod(FieldInfo fieldInfo) - { -#if SIMPLE_JSON_NO_LINQ_EXPRESSION - return GetSetMethodByReflection(fieldInfo); -#else - return GetSetMethodByExpression(fieldInfo); -#endif - } - - public static SetDelegate GetSetMethodByReflection(PropertyInfo propertyInfo) - { - MethodInfo methodInfo = GetSetterMethodInfo(propertyInfo); - return delegate(object source, object value) { methodInfo.Invoke(source, new object[] { value }); }; - } - - public static SetDelegate GetSetMethodByReflection(FieldInfo fieldInfo) - { - return delegate(object source, object value) { fieldInfo.SetValue(source, value); }; - } - -#if !SIMPLE_JSON_NO_LINQ_EXPRESSION - - public static SetDelegate GetSetMethodByExpression(PropertyInfo propertyInfo) - { - MethodInfo setMethodInfo = GetSetterMethodInfo(propertyInfo); - ParameterExpression instance = Expression.Parameter(typeof(object), "instance"); - ParameterExpression value = Expression.Parameter(typeof(object), "value"); - UnaryExpression instanceCast = (!IsValueType(propertyInfo.DeclaringType)) ? Expression.TypeAs(instance, propertyInfo.DeclaringType) : Expression.Convert(instance, propertyInfo.DeclaringType); - UnaryExpression valueCast = (!IsValueType(propertyInfo.PropertyType)) ? Expression.TypeAs(value, propertyInfo.PropertyType) : Expression.Convert(value, propertyInfo.PropertyType); - Action compiled = Expression.Lambda>(Expression.Call(instanceCast, setMethodInfo, valueCast), new ParameterExpression[] { instance, value }).Compile(); - return delegate(object source, object val) { compiled(source, val); }; - } - - public static SetDelegate GetSetMethodByExpression(FieldInfo fieldInfo) - { - ParameterExpression instance = Expression.Parameter(typeof(object), "instance"); - ParameterExpression value = Expression.Parameter(typeof(object), "value"); - Action compiled = Expression.Lambda>( - Assign(Expression.Field(Expression.Convert(instance, fieldInfo.DeclaringType), fieldInfo), Expression.Convert(value, fieldInfo.FieldType)), instance, value).Compile(); - return delegate(object source, object val) { compiled(source, val); }; - } - - public static BinaryExpression Assign(Expression left, Expression right) - { -#if SIMPLE_JSON_TYPEINFO - return Expression.Assign(left, right); -#else - MethodInfo assign = typeof(Assigner<>).MakeGenericType(left.Type).GetMethod("Assign"); - BinaryExpression assignExpr = Expression.Add(left, right, assign); - return assignExpr; -#endif - } - - private static class Assigner - { - public static T Assign(ref T left, T right) - { - return (left = right); - } - } - -#endif - - public sealed class ThreadSafeDictionary : IDictionary - { - private readonly object _lock = new object(); - private readonly ThreadSafeDictionaryValueFactory _valueFactory; - private Dictionary _dictionary; - - public ThreadSafeDictionary(ThreadSafeDictionaryValueFactory valueFactory) - { - _valueFactory = valueFactory; - } - - private TValue Get(TKey key) - { - if (_dictionary == null) - return AddValue(key); - TValue value; - if (!_dictionary.TryGetValue(key, out value)) - return AddValue(key); - return value; - } - - private TValue AddValue(TKey key) - { - TValue value = _valueFactory(key); - lock (_lock) - { - if (_dictionary == null) - { - _dictionary = new Dictionary(); - _dictionary[key] = value; - } - else - { - TValue val; - if (_dictionary.TryGetValue(key, out val)) - return val; - Dictionary dict = new Dictionary(_dictionary); - dict[key] = value; - _dictionary = dict; - } - } - return value; - } - - public void Add(TKey key, TValue value) - { - throw new NotImplementedException(); - } - - public bool ContainsKey(TKey key) - { - return _dictionary.ContainsKey(key); - } - - public ICollection Keys - { - get { return _dictionary.Keys; } - } - - public bool Remove(TKey key) - { - throw new NotImplementedException(); - } - - public bool TryGetValue(TKey key, out TValue value) - { - value = this[key]; - return true; - } - - public ICollection Values - { - get { return _dictionary.Values; } - } - - public TValue this[TKey key] - { - get { return Get(key); } - set { throw new NotImplementedException(); } - } - - public void Add(KeyValuePair item) - { - throw new NotImplementedException(); - } - - public void Clear() - { - throw new NotImplementedException(); - } - - public bool Contains(KeyValuePair item) - { - throw new NotImplementedException(); - } - - public void CopyTo(KeyValuePair[] array, int arrayIndex) - { - throw new NotImplementedException(); - } - - public int Count - { - get { return _dictionary.Count; } - } - - public bool IsReadOnly - { - get { throw new NotImplementedException(); } - } - - public bool Remove(KeyValuePair item) - { - throw new NotImplementedException(); - } - - public IEnumerator> GetEnumerator() - { - return _dictionary.GetEnumerator(); - } - - System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() - { - return _dictionary.GetEnumerator(); - } - } - - } - } -} diff --git a/src/NServiceBus.AwsLambda.SQS/TransportMessage.cs b/src/NServiceBus.AwsLambda.SQS/TransportMessage.cs index a37fc26..b38de6e 100644 --- a/src/NServiceBus.AwsLambda.SQS/TransportMessage.cs +++ b/src/NServiceBus.AwsLambda.SQS/TransportMessage.cs @@ -6,6 +6,9 @@ class TransportMessage { + //MessageBody of Amazon.SQS SendRequest must have a minimum length of 1: https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_SendMessage.html + public const string EmptyMessage = "empty message"; + // Empty constructor required for deserialization. public TransportMessage() { @@ -27,7 +30,18 @@ public TransportMessage(OutgoingMessage outgoingMessage, DispatchProperties prop TimeToBeReceived = properties.DiscardIfNotReceivedBefore.MaxTime.ToString(); } - Body = !outgoingMessage.Body.IsEmpty ? Convert.ToBase64String(outgoingMessage.Body.Span) : "empty message"; + if (outgoingMessage.Body.Length != 0) + { +#if NETFRAMEWORK + Body = Convert.ToBase64String(outgoingMessage.Body.ToArray()); +#else + Body = Convert.ToBase64String(outgoingMessage.Body.Span); +#endif + } + else + { + Body = EmptyMessage; + } } public Dictionary Headers { get; set; } diff --git a/src/NServiceBus.AwsLambda.SQS/TransportMessageSerializerContext.cs b/src/NServiceBus.AwsLambda.SQS/TransportMessageSerializerContext.cs new file mode 100644 index 0000000..41eccfb --- /dev/null +++ b/src/NServiceBus.AwsLambda.SQS/TransportMessageSerializerContext.cs @@ -0,0 +1,10 @@ +namespace NServiceBus.AwsLambda.SQS +{ + using System.Text.Json.Serialization; + + [JsonSourceGenerationOptions] + [JsonSerializable(typeof(TransportMessage))] + partial class TransportMessageSerializerContext : JsonSerializerContext + { + } +} \ No newline at end of file From fd29fec968c5e138b90475ad12b30de66f2ea22d Mon Sep 17 00:00:00 2001 From: danielmarbach Date: Tue, 4 Apr 2023 11:18:26 +0200 Subject: [PATCH 25/27] Better integration with the transport and native integration --- .../AwsLambdaEndpoint.cs | 129 ++++++++++++------ .../ExceptionExtensions.cs | 12 ++ .../MessageExtensions.cs | 38 +----- .../S3EncryptionMethodExtensions.cs | 21 +++ .../TransportHeaders.cs | 3 + .../TransportMessageExtensions.cs | 98 +++++++++++++ 6 files changed, 231 insertions(+), 70 deletions(-) create mode 100644 src/NServiceBus.AwsLambda.SQS/ExceptionExtensions.cs create mode 100644 src/NServiceBus.AwsLambda.SQS/S3EncryptionMethodExtensions.cs create mode 100644 src/NServiceBus.AwsLambda.SQS/TransportMessageExtensions.cs diff --git a/src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs b/src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs index 8175cb3..be040d3 100644 --- a/src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs +++ b/src/NServiceBus.AwsLambda.SQS/AwsLambdaEndpoint.cs @@ -1,13 +1,13 @@ namespace NServiceBus { using System; + using System.Buffers; using System.Collections.Generic; using System.Text.Json; using System.Threading; using System.Threading.Tasks; using Amazon.Lambda.Core; using Amazon.Lambda.SQSEvents; - using Amazon.S3; using Amazon.SQS; using Amazon.SQS.Model; using AwsLambda.SQS; @@ -202,12 +202,7 @@ async Task Initialize(AwsLambdaSQSEndpointConfiguration con errorQueueUrl = await GetQueueUrl(settingsHolder.ErrorQueueAddress()) .ConfigureAwait(false); - s3BucketForLargeMessages = configuration.Transport.S3?.BucketName; - - if (!string.IsNullOrWhiteSpace(s3BucketForLargeMessages)) - { - s3Client = configuration.Transport.S3?.S3Client; - } + s3Settings = configuration.Transport.S3; var serverlessTransport = new ServerlessTransport(configuration.Transport); configuration.EndpointConfiguration.UseTransport(serverlessTransport); @@ -231,7 +226,9 @@ async Task GetQueueUrl(string queueName) async Task ProcessMessage(SQSEvent.SQSMessage receivedMessage, ILambdaContext lambdaContext, CancellationToken token) { - byte[] messageBody = null; + var arrayPool = ArrayPool.Shared; + ReadOnlyMemory messageBody = null; + byte[] messageBodyBuffer = null; TransportMessage transportMessage = null; Exception exception = null; var nativeMessageId = receivedMessage.MessageId; @@ -240,44 +237,101 @@ async Task ProcessMessage(SQSEvent.SQSMessage receivedMessage, ILambdaContext la try { - if (receivedMessage.MessageAttributes.TryGetValue(Headers.MessageId, out var messageIdAttribute)) + try { - messageId = messageIdAttribute.StringValue; + if (receivedMessage.MessageAttributes.TryGetValue(Headers.MessageId, out var messageIdAttribute)) + { + messageId = messageIdAttribute.StringValue; + } + else + { + messageId = nativeMessageId; + } + + if (receivedMessage.MessageAttributes.TryGetValue(TransportHeaders.Headers, out var headersAttribute)) + { + transportMessage = new TransportMessage + { + Headers = JsonSerializer.Deserialize>(headersAttribute.StringValue) ?? new Dictionary(), + Body = receivedMessage.Body + }; + transportMessage.Headers[Headers.MessageId] = messageId; + if (receivedMessage.MessageAttributes.TryGetValue(TransportHeaders.S3BodyKey, out var s3BodyKey)) + { + transportMessage.Headers[TransportHeaders.S3BodyKey] = s3BodyKey.StringValue; + transportMessage.S3BodyKey = s3BodyKey.StringValue; + } + } + else + { + // When the MessageTypeFullName attribute is available, we're assuming native integration + if (receivedMessage.MessageAttributes.TryGetValue(TransportHeaders.MessageTypeFullName, + out var enclosedMessageType)) + { + var headers = new Dictionary + { + { Headers.MessageId, messageId }, + { Headers.EnclosedMessageTypes, enclosedMessageType.StringValue }, + { + TransportHeaders.MessageTypeFullName, enclosedMessageType.StringValue + } // we're copying over the value of the native message attribute into the headers, converting this into a nsb message + }; + + if (receivedMessage.MessageAttributes.TryGetValue(TransportHeaders.S3BodyKey, + out var s3BodyKey)) + { + headers.Add(TransportHeaders.S3BodyKey, s3BodyKey.StringValue); + } + + transportMessage = new TransportMessage + { + Headers = headers, + S3BodyKey = s3BodyKey?.StringValue, + Body = receivedMessage.Body + }; + } + else + { + transportMessage = JsonSerializer.Deserialize(receivedMessage.Body, + transportMessageSerializerOptions); + } + } + + (messageBody, messageBodyBuffer) = await transportMessage.RetrieveBody(messageId, s3Settings, arrayPool, token).ConfigureAwait(false); } - else + catch (Exception ex) when (!ex.IsCausedBy(token)) { - messageId = nativeMessageId; + // Can't deserialize. This is a poison message + exception = ex; + isPoisonMessage = true; } - transportMessage = JsonSerializer.Deserialize(receivedMessage.Body, transportMessageSerializerOptions); + if (isPoisonMessage || transportMessage == null) + { + LogPoisonMessage(messageId, exception); - messageBody = await transportMessage.RetrieveBody(s3Client, s3BucketForLargeMessages, token).ConfigureAwait(false); - } - catch (Exception ex) when (ex is not OperationCanceledException) - { - // Can't deserialize. This is a poison message - exception = ex; - isPoisonMessage = true; - } + await MovePoisonMessageToErrorQueue(receivedMessage, messageId).ConfigureAwait(false); + return; + } - if (isPoisonMessage || messageBody == null || transportMessage == null) - { - LogPoisonMessage(messageId, exception); + if (!IsMessageExpired(receivedMessage, transportMessage.Headers, messageId, sqsClient.Config.ClockOffset)) + { + // here we also want to use the native message id because the core demands it like that + await ProcessMessageWithInMemoryRetries(transportMessage.Headers, nativeMessageId, messageBody, lambdaContext, token).ConfigureAwait(false); + } - await MovePoisonMessageToErrorQueue(receivedMessage, messageId).ConfigureAwait(false); - return; + // Always delete the message from the queue. + // If processing failed, the onError handler will have moved the message + // to a retry queue. + await DeleteMessageAndBodyIfRequired(receivedMessage, transportMessage.S3BodyKey).ConfigureAwait(false); } - - if (!IsMessageExpired(receivedMessage, transportMessage.Headers, messageId, sqsClient.Config.ClockOffset)) + finally { - // here we also want to use the native message id because the core demands it like that - await ProcessMessageWithInMemoryRetries(transportMessage.Headers, nativeMessageId, messageBody, lambdaContext, token).ConfigureAwait(false); + if (messageBodyBuffer != null) + { + arrayPool.Return(messageBodyBuffer, clearArray: true); + } } - - // Always delete the message from the queue. - // If processing failed, the onError handler will have moved the message - // to a retry queue. - await DeleteMessageAndBodyIfRequired(receivedMessage, transportMessage.S3BodyKey).ConfigureAwait(false); } static bool IsMessageExpired(SQSEvent.SQSMessage receivedMessage, Dictionary headers, string messageId, TimeSpan clockOffset) @@ -294,7 +348,7 @@ static bool IsMessageExpired(SQSEvent.SQSMessage receivedMessage, Dictionary utcNow) @@ -481,8 +535,7 @@ static void LogPoisonMessage(string messageId, Exception exception) PipelineInvoker pipeline; IEndpointInstance endpoint; IAmazonSQS sqsClient; - IAmazonS3 s3Client; - string s3BucketForLargeMessages; + S3Settings s3Settings; string queueUrl; string errorQueueUrl; diff --git a/src/NServiceBus.AwsLambda.SQS/ExceptionExtensions.cs b/src/NServiceBus.AwsLambda.SQS/ExceptionExtensions.cs new file mode 100644 index 0000000..3c4d3b4 --- /dev/null +++ b/src/NServiceBus.AwsLambda.SQS/ExceptionExtensions.cs @@ -0,0 +1,12 @@ +namespace NServiceBus +{ + using System; + using System.Threading; + + static class ExceptionExtensions + { +#pragma warning disable PS0003 // A parameter of type CancellationToken on a non-private delegate or method should be optional + public static bool IsCausedBy(this Exception ex, CancellationToken cancellationToken) => ex is OperationCanceledException && cancellationToken.IsCancellationRequested; +#pragma warning restore PS0003 // A parameter of type CancellationToken on a non-private delegate or method should be optional + } +} \ No newline at end of file diff --git a/src/NServiceBus.AwsLambda.SQS/MessageExtensions.cs b/src/NServiceBus.AwsLambda.SQS/MessageExtensions.cs index 16ed0e3..e653985 100644 --- a/src/NServiceBus.AwsLambda.SQS/MessageExtensions.cs +++ b/src/NServiceBus.AwsLambda.SQS/MessageExtensions.cs @@ -1,47 +1,21 @@ -namespace NServiceBus.AwsLambda.SQS +#nullable enable + +namespace NServiceBus.AwsLambda.SQS { using System; using System.Globalization; - using System.IO; - using System.Threading; - using System.Threading.Tasks; using Amazon.Lambda.SQSEvents; - using Amazon.S3; static class MessageExtensions { - public static async Task RetrieveBody(this TransportMessage transportMessage, - IAmazonS3 s3Client, - string s3BucketForLargeMessages, - CancellationToken cancellationToken) - { - if (string.IsNullOrEmpty(transportMessage.S3BodyKey)) - { - return Convert.FromBase64String(transportMessage.Body); - } - - if (s3Client == null) - { - throw new Exception("Unable to retrieve the body from S3. Configure the bucket name and key prefix with `transport.S3(string bucketForLargeMessages, string keyPrefix)`"); - } - - var s3GetResponse = await s3Client.GetObjectAsync(s3BucketForLargeMessages, - transportMessage.S3BodyKey, - cancellationToken).ConfigureAwait(false); - - using var memoryStream = new MemoryStream(); - await s3GetResponse.ResponseStream.CopyToAsync(memoryStream, cancellationToken).ConfigureAwait(false); - return memoryStream.ToArray(); - } - - public static DateTime GetAdjustedDateTimeFromServerSetAttributes(this SQSEvent.SQSMessage message, TimeSpan clockOffset) + public static DateTimeOffset GetAdjustedDateTimeFromServerSetAttributes(this SQSEvent.SQSMessage message, string attributeName, TimeSpan clockOffset) { - var result = UnixEpoch.AddMilliseconds(long.Parse(message.Attributes["SentTimestamp"], NumberFormatInfo.InvariantInfo)); + var result = UnixEpoch.AddMilliseconds(long.Parse(message.Attributes[attributeName], NumberFormatInfo.InvariantInfo)); // Adjust for clock skew between this endpoint and aws. // https://aws.amazon.com/blogs/developer/clock-skew-correction/ return result + clockOffset; } - static readonly DateTime UnixEpoch = new(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); + static readonly DateTimeOffset UnixEpoch = new DateTimeOffset(1970, 1, 1, 0, 0, 0, TimeSpan.Zero); } } \ No newline at end of file diff --git a/src/NServiceBus.AwsLambda.SQS/S3EncryptionMethodExtensions.cs b/src/NServiceBus.AwsLambda.SQS/S3EncryptionMethodExtensions.cs new file mode 100644 index 0000000..cbb1ef9 --- /dev/null +++ b/src/NServiceBus.AwsLambda.SQS/S3EncryptionMethodExtensions.cs @@ -0,0 +1,21 @@ +#nullable enable +namespace NServiceBus.AwsLambda.SQS +{ + using System.Reflection; + using Amazon.S3.Model; + + static class S3EncryptionMethodExtensions + { + public static void ModifyRequest(this S3EncryptionMethod encryptionMethod, GetObjectRequest request) + { + if (encryptionMethod == null) + { + return; + } + + // TODO: Optimize + var methodInfo = encryptionMethod.GetType().GetMethod("ModifyGetRequest", BindingFlags.NonPublic | BindingFlags.Instance); + methodInfo!.Invoke(encryptionMethod, new object?[] { request }); + } + } +} \ No newline at end of file diff --git a/src/NServiceBus.AwsLambda.SQS/TransportHeaders.cs b/src/NServiceBus.AwsLambda.SQS/TransportHeaders.cs index 391f80c..91f0d7d 100644 --- a/src/NServiceBus.AwsLambda.SQS/TransportHeaders.cs +++ b/src/NServiceBus.AwsLambda.SQS/TransportHeaders.cs @@ -5,5 +5,8 @@ static class TransportHeaders const string Prefix = "NServiceBus.AmazonSQS."; public const string TimeToBeReceived = Prefix + nameof(TimeToBeReceived); public const string DelaySeconds = Prefix + nameof(DelaySeconds); + public const string Headers = Prefix + nameof(Headers); + public const string S3BodyKey = "S3BodyKey"; + public const string MessageTypeFullName = "MessageTypeFullName"; } } \ No newline at end of file diff --git a/src/NServiceBus.AwsLambda.SQS/TransportMessageExtensions.cs b/src/NServiceBus.AwsLambda.SQS/TransportMessageExtensions.cs new file mode 100644 index 0000000..567bbb5 --- /dev/null +++ b/src/NServiceBus.AwsLambda.SQS/TransportMessageExtensions.cs @@ -0,0 +1,98 @@ +#nullable enable +namespace NServiceBus.AwsLambda.SQS +{ + using System; + using System.Buffers; + using System.IO; + using System.Linq; + using System.Text; + using System.Threading; + using System.Threading.Tasks; + using Amazon.S3.Model; + + static class TransportMessageExtensions + { + public static async ValueTask<(ReadOnlyMemory MessageBody, byte[]? MessageBodyBuffer)> RetrieveBody(this TransportMessage transportMessage, string messageId, S3Settings s3Settings, ArrayPool arrayPool, + CancellationToken cancellationToken = default) + { + if (string.IsNullOrEmpty(transportMessage.S3BodyKey)) + { + if (transportMessage.Body == TransportMessage.EmptyMessage) + { + return EmptyMessage; + } + + return ConvertBody(transportMessage.Body, arrayPool, transportMessage.Headers.Keys.Contains(TransportHeaders.Headers)); + } + + if (s3Settings == null) + { + throw new Exception($"The message {messageId} contains the ID of the body stored in S3 but this endpoint is not configured to use S3 for body storage."); + } + + var getObjectRequest = new GetObjectRequest + { + BucketName = s3Settings.BucketName, + Key = transportMessage.S3BodyKey + }; + + s3Settings.Encryption?.ModifyRequest(getObjectRequest); + + var s3GetResponse = await s3Settings.S3Client.GetObjectAsync(getObjectRequest, cancellationToken) + .ConfigureAwait(false); + + int contentLength = (int)s3GetResponse.ContentLength; + var buffer = arrayPool.Rent(contentLength); + using var memoryStream = new MemoryStream(buffer); + await s3GetResponse.ResponseStream.CopyToAsync(memoryStream, 81920, cancellationToken).ConfigureAwait(false); + return (buffer.AsMemory(0, contentLength), buffer); + } + + static (ReadOnlyMemory MessageBody, byte[]? MessageBodyBuffer) ConvertBody(string body, ArrayPool arrayPool, bool isNativeMessage) + { + var encoding = Encoding.UTF8; + + if (isNativeMessage) + { + return GetNonEncodedBody(body, arrayPool, null, encoding); + } + else + { +#if NETFRAMEWORK + try + { + return (Convert.FromBase64String(body), null); + } + catch (FormatException) + { + return GetNonEncodedBody(body, arrayPool, null, encoding); + } +#else + var buffer = GetBuffer(body, arrayPool, encoding); + if (Convert.TryFromBase64String(body, buffer, out var writtenBytes)) + { + return (buffer.AsMemory(0, writtenBytes), buffer); + } + + return GetNonEncodedBody(body, arrayPool, buffer, encoding); +#endif + } + } + + static (ReadOnlyMemory MessageBody, byte[]? MessageBodyBuffer) GetNonEncodedBody(string body, ArrayPool arrayPool, byte[]? buffer, Encoding encoding) + { + buffer ??= GetBuffer(body, arrayPool, encoding); + var writtenBytes = encoding.GetBytes(body, 0, body.Length, buffer, 0); + return (buffer.AsMemory(0, writtenBytes), buffer); + } + + static byte[] GetBuffer(string body, ArrayPool arrayPool, Encoding encoding) + { + var length = encoding.GetMaxByteCount(body.Length); + return arrayPool.Rent(length); + } + + static readonly (ReadOnlyMemory MessageBody, byte[]? MessageBodyBuffer) + EmptyMessage = (Array.Empty(), null); + } +} \ No newline at end of file From fe00e3782a4dc85c51fc8b28ec7376caf7cc54a0 Mon Sep 17 00:00:00 2001 From: danielmarbach Date: Tue, 4 Apr 2023 12:00:36 +0200 Subject: [PATCH 26/27] Properly testing native integration --- .../AwsLambdaSQSEndpointTestBase.cs | 40 +++- .../MessageExtensions.cs | 2 +- ...ceBus.AwsLambda.SQS.AcceptanceTests.csproj | 2 +- ...eceiving_a_native_message_with_encoding.cs | 142 ++++++++++++++ ...eiving_a_native_message_without_wrapper.cs | 175 ++++++++++++++++++ .../ReceiveMessageResponseExtensions.cs | 2 +- ...SEvent_with_large_payloads_is_processed.cs | 4 +- .../When_a_handler_sends_a_message.cs | 2 +- .../When_a_message_handler_always_throws.cs | 2 +- .../When_the_SQS_trigger_is_activated.cs | 2 +- 10 files changed, 363 insertions(+), 10 deletions(-) create mode 100644 src/NServiceBus.AwsLambda.SQS.AcceptanceTests/NativeIntegration/When_receiving_a_native_message_with_encoding.cs create mode 100644 src/NServiceBus.AwsLambda.SQS.AcceptanceTests/NativeIntegration/When_receiving_a_native_message_without_wrapper.cs diff --git a/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/AwsLambdaSQSEndpointTestBase.cs b/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/AwsLambdaSQSEndpointTestBase.cs index 9691cf5..a691ed4 100644 --- a/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/AwsLambdaSQSEndpointTestBase.cs +++ b/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/AwsLambdaSQSEndpointTestBase.cs @@ -1,10 +1,11 @@ -namespace NServiceBus.AwsLambda.Tests +namespace NServiceBus.AcceptanceTests { using System; using System.Collections.Generic; using System.Globalization; using System.IO; using System.Linq; + using System.Text; using System.Threading.Tasks; using Amazon.Lambda.SQSEvents; using Amazon.Runtime; @@ -133,6 +134,8 @@ protected async Task GenerateAndReceiveSQSEvent(int count) where T { MaxNumberOfMessages = count, WaitTimeSeconds = 20, + AttributeNames = new List { "SentTimestamp" }, + MessageAttributeNames = new List { "*" } }; var receivedMessages = await sqsClient.ReceiveMessageAsync(receiveRequest); @@ -140,6 +143,41 @@ protected async Task GenerateAndReceiveSQSEvent(int count) where T return receivedMessages.ToSQSEvent(); } + protected async Task GenerateAndReceiveNativeSQSEvent(Dictionary messageAttributeValues, string message, bool base64Encode = true) + { + var body = base64Encode ? Convert.ToBase64String(Encoding.UTF8.GetBytes(message)) : message; + + var sendMessageRequest = new SendMessageRequest + { + QueueUrl = createdQueue.QueueUrl, + MessageAttributes = messageAttributeValues, + MessageBody = body + }; + + await sqsClient.SendMessageAsync(sendMessageRequest) + .ConfigureAwait(false); + + var receiveRequest = new ReceiveMessageRequest(createdQueue.QueueUrl) + { + MaxNumberOfMessages = 10, + WaitTimeSeconds = 20, + AttributeNames = new List { "SentTimestamp" }, + MessageAttributeNames = new List { "*" } + }; + + var receivedMessages = await sqsClient.ReceiveMessageAsync(receiveRequest); + + return receivedMessages.ToSQSEvent(); + } + + protected async Task UploadMessageBodyToS3(string key, string body) => + await s3Client.PutObjectAsync(new PutObjectRequest + { + Key = $"{key}", + BucketName = BucketName, + ContentBody = body + }); + protected async Task CountMessagesInErrorQueue() { var attReq = new GetQueueAttributesRequest { QueueUrl = createdErrorQueue.QueueUrl }; diff --git a/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/MessageExtensions.cs b/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/MessageExtensions.cs index 2c7a76d..0163f4f 100644 --- a/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/MessageExtensions.cs +++ b/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/MessageExtensions.cs @@ -1,4 +1,4 @@ -namespace NServiceBus.AwsLambda.Tests +namespace NServiceBus.AcceptanceTests { using System.Linq; using Amazon.Lambda.SQSEvents; diff --git a/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/NServiceBus.AwsLambda.SQS.AcceptanceTests.csproj b/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/NServiceBus.AwsLambda.SQS.AcceptanceTests.csproj index 972c368..9b82e1d 100644 --- a/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/NServiceBus.AwsLambda.SQS.AcceptanceTests.csproj +++ b/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/NServiceBus.AwsLambda.SQS.AcceptanceTests.csproj @@ -3,7 +3,7 @@ net6.0;net7.0 true - NServiceBus.AwsLambda.Tests + NServiceBus.AcceptanceTests diff --git a/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/NativeIntegration/When_receiving_a_native_message_with_encoding.cs b/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/NativeIntegration/When_receiving_a_native_message_with_encoding.cs new file mode 100644 index 0000000..ca90b35 --- /dev/null +++ b/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/NativeIntegration/When_receiving_a_native_message_with_encoding.cs @@ -0,0 +1,142 @@ +namespace NServiceBus.AcceptanceTests.NativeIntegration +{ + using System; + using System.Collections.Generic; + using System.Threading.Tasks; + using System.Xml.Linq; + using Amazon.SQS.Model; + using Microsoft.Extensions.DependencyInjection; + using NServiceBus.AcceptanceTests; + using NUnit.Framework; + + class When_receiving_a_native_message_with_encoding : AwsLambdaSQSEndpointTestBase + { + static readonly string MessageToSend = new XDocument(new XElement("Message", new XElement("ThisIsTheMessage", "Hello!"))).ToString(); + + [Test] + public async Task Should_be_processed_when_messagetypefullname_present() + { + var receivedMessages = await GenerateAndReceiveNativeSQSEvent(new Dictionary + { + {"MessageTypeFullName", new MessageAttributeValue {DataType = "String", StringValue = typeof(Message).FullName}} + }, MessageToSend); + + var context = new TestContext(); + + var endpoint = new AwsLambdaSQSEndpoint(ctx => + { + var configuration = new AwsLambdaSQSEndpointConfiguration(QueueName, CreateSQSClient(), CreateSNSClient()); + var transport = configuration.Transport; + + transport.S3 = new S3Settings(BucketName, KeyPrefix, CreateS3Client()); + + var advanced = configuration.AdvancedConfiguration; + advanced.SendFailedMessagesTo(ErrorQueueName); + advanced.RegisterComponents(c => c.AddSingleton(typeof(TestContext), context)); + return configuration; + }); + + await endpoint.Process(receivedMessages, null); + + Assert.AreEqual("Hello!", context.MessageReceived); + + var messagesInErrorQueueCount = await CountMessagesInErrorQueue(); + + Assert.AreEqual(0, messagesInErrorQueueCount); + } + + [Test] + public async Task Should_fail_when_messagetypefullname_not_present() + { + var messageId = Guid.NewGuid(); + var receivedMessages = await GenerateAndReceiveNativeSQSEvent(new Dictionary + { + // unfortunately only the message id attribute is preserved when moving to the poison queue + { + Headers.MessageId, new MessageAttributeValue {DataType = "String", StringValue = messageId.ToString() } + } + }, MessageToSend); + + var context = new TestContext(); + + var endpoint = new AwsLambdaSQSEndpoint(ctx => + { + var configuration = new AwsLambdaSQSEndpointConfiguration(QueueName, CreateSQSClient(), CreateSNSClient()); + var transport = configuration.Transport; + + transport.S3 = new S3Settings(BucketName, KeyPrefix, CreateS3Client()); + + var advanced = configuration.AdvancedConfiguration; + advanced.SendFailedMessagesTo(ErrorQueueName); + advanced.RegisterComponents(c => c.AddSingleton(typeof(TestContext), context)); + return configuration; + }); + + await endpoint.Process(receivedMessages, null); + + var messagesInErrorQueueCount = await CountMessagesInErrorQueue(); + + Assert.AreEqual(1, messagesInErrorQueueCount); + } + + [Test] + public async Task Should_support_loading_body_from_s3() + { + var s3Key = Guid.NewGuid().ToString(); + + await UploadMessageBodyToS3(s3Key, MessageToSend); + + var receivedMessages = await GenerateAndReceiveNativeSQSEvent(new Dictionary + { + {"MessageTypeFullName", new MessageAttributeValue {DataType = "String", StringValue = typeof(Message).FullName}}, + {"S3BodyKey", new MessageAttributeValue {DataType = "String", StringValue = s3Key}}, + }, MessageToSend); + + var context = new TestContext(); + + var endpoint = new AwsLambdaSQSEndpoint(ctx => + { + var configuration = new AwsLambdaSQSEndpointConfiguration(QueueName, CreateSQSClient(), CreateSNSClient()); + var transport = configuration.Transport; + + transport.S3 = new S3Settings(BucketName, KeyPrefix, CreateS3Client()); + + var advanced = configuration.AdvancedConfiguration; + advanced.SendFailedMessagesTo(ErrorQueueName); + advanced.RegisterComponents(c => c.AddSingleton(typeof(TestContext), context)); + return configuration; + }); + + await endpoint.Process(receivedMessages, null); + + Assert.AreEqual("Hello!", context.MessageReceived); + + var messagesInErrorQueueCount = await CountMessagesInErrorQueue(); + + Assert.AreEqual(0, messagesInErrorQueueCount); + } + + public class TestContext + { + public string MessageReceived { get; set; } + } + + public class Message : IMessage + { + public string ThisIsTheMessage { get; set; } + } + + public class WithEncodingHandler : IHandleMessages + { + public WithEncodingHandler(TestContext context) => testContext = context; + + public Task Handle(Message message, IMessageHandlerContext context) + { + testContext.MessageReceived = message.ThisIsTheMessage; + return Task.CompletedTask; + } + + TestContext testContext; + } + } +} \ No newline at end of file diff --git a/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/NativeIntegration/When_receiving_a_native_message_without_wrapper.cs b/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/NativeIntegration/When_receiving_a_native_message_without_wrapper.cs new file mode 100644 index 0000000..4e87395 --- /dev/null +++ b/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/NativeIntegration/When_receiving_a_native_message_without_wrapper.cs @@ -0,0 +1,175 @@ +namespace NServiceBus.AcceptanceTests.NativeIntegration +{ + using System; + using System.Collections.Generic; + using System.Text.Json; + using System.Threading.Tasks; + using System.Xml.Linq; + using Amazon.SQS.Model; + using Microsoft.Extensions.DependencyInjection; + using NServiceBus.AcceptanceTests; + using NUnit.Framework; + + class When_receiving_a_native_message_without_wrapper : AwsLambdaSQSEndpointTestBase + { + static readonly string MessageToSend = new XDocument(new XElement("NServiceBus.AcceptanceTests.NativeIntegration.NativeMessage", new XElement("ThisIsTheMessage", "Hello!"))).ToString(); + + [Test] + public async Task Should_be_processed_when_nsbheaders_present_with_messageid() + { + var receivedMessages = await GenerateAndReceiveNativeSQSEvent(new Dictionary + { + { + "NServiceBus.AmazonSQS.Headers", + new MessageAttributeValue + { + DataType = "String", StringValue = GetHeaders(messageId: Guid.NewGuid().ToString()) + } + } + }, MessageToSend, base64Encode: false); + + var context = new TestContext(); + + var endpoint = new AwsLambdaSQSEndpoint(ctx => + { + var configuration = new AwsLambdaSQSEndpointConfiguration(QueueName, CreateSQSClient(), CreateSNSClient()); + var transport = configuration.Transport; + + transport.S3 = new S3Settings(BucketName, KeyPrefix, CreateS3Client()); + + var advanced = configuration.AdvancedConfiguration; + advanced.SendFailedMessagesTo(ErrorQueueName); + advanced.RegisterComponents(c => c.AddSingleton(typeof(TestContext), context)); + return configuration; + }); + + await endpoint.Process(receivedMessages, null); + + Assert.AreEqual("Hello!", context.MessageReceived); + + var messagesInErrorQueueCount = await CountMessagesInErrorQueue(); + + Assert.AreEqual(0, messagesInErrorQueueCount); + } + + [Test] + public async Task Should_be_processed_when_nsbheaders_present_without_messageid() + { + var receivedMessages = await GenerateAndReceiveNativeSQSEvent(new Dictionary + { + { + "NServiceBus.AmazonSQS.Headers", + new MessageAttributeValue + { + DataType = "String", StringValue = GetHeaders() + } + } + }, MessageToSend, base64Encode: false); + + var context = new TestContext(); + + var endpoint = new AwsLambdaSQSEndpoint(ctx => + { + var configuration = new AwsLambdaSQSEndpointConfiguration(QueueName, CreateSQSClient(), CreateSNSClient()); + var transport = configuration.Transport; + + transport.S3 = new S3Settings(BucketName, KeyPrefix, CreateS3Client()); + + var advanced = configuration.AdvancedConfiguration; + advanced.SendFailedMessagesTo(ErrorQueueName); + advanced.RegisterComponents(c => c.AddSingleton(typeof(TestContext), context)); + return configuration; + }); + + await endpoint.Process(receivedMessages, null); + + Assert.AreEqual("Hello!", context.MessageReceived); + + var messagesInErrorQueueCount = await CountMessagesInErrorQueue(); + + Assert.AreEqual(0, messagesInErrorQueueCount); + } + + [Test] + public async Task Should_support_loading_body_from_s3() + { + var s3Key = Guid.NewGuid().ToString(); + + await UploadMessageBodyToS3(s3Key, MessageToSend); + + var receivedMessages = await GenerateAndReceiveNativeSQSEvent(new Dictionary + { + { + "NServiceBus.AmazonSQS.Headers", + new MessageAttributeValue + { + DataType = "String", StringValue = GetHeaders(s3Key: s3Key) + } + } + }, MessageToSend, base64Encode: false); + + var context = new TestContext(); + + var endpoint = new AwsLambdaSQSEndpoint(ctx => + { + var configuration = new AwsLambdaSQSEndpointConfiguration(QueueName, CreateSQSClient(), CreateSNSClient()); + var transport = configuration.Transport; + + transport.S3 = new S3Settings(BucketName, KeyPrefix, CreateS3Client()); + + var advanced = configuration.AdvancedConfiguration; + advanced.SendFailedMessagesTo(ErrorQueueName); + advanced.RegisterComponents(c => c.AddSingleton(typeof(TestContext), context)); + return configuration; + }); + + await endpoint.Process(receivedMessages, null); + + Assert.AreEqual("Hello!", context.MessageReceived); + + var messagesInErrorQueueCount = await CountMessagesInErrorQueue(); + + Assert.AreEqual(0, messagesInErrorQueueCount); + } + + string GetHeaders(string s3Key = null, string messageId = null) + { + var nsbHeaders = new Dictionary(); + + if (!string.IsNullOrEmpty(s3Key)) + { + nsbHeaders.Add("S3BodyKey", "s3Key"); + } + + if (!string.IsNullOrEmpty(messageId)) + { + nsbHeaders.Add("NServiceBus.MessageId", messageId); + } + + return JsonSerializer.Serialize(nsbHeaders); + } + + public class TestContext + { + public string MessageReceived { get; set; } + } + + public class NativeHandler : IHandleMessages + { + public NativeHandler(TestContext context) => testContext = context; + + public Task Handle(NativeMessage message, IMessageHandlerContext context) + { + testContext.MessageReceived = message.ThisIsTheMessage; + return Task.CompletedTask; + } + + readonly TestContext testContext; + } + } + + public class NativeMessage : IMessage + { + public string ThisIsTheMessage { get; set; } + } +} \ No newline at end of file diff --git a/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/ReceiveMessageResponseExtensions.cs b/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/ReceiveMessageResponseExtensions.cs index 8f93026..a510479 100644 --- a/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/ReceiveMessageResponseExtensions.cs +++ b/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/ReceiveMessageResponseExtensions.cs @@ -1,4 +1,4 @@ -namespace NServiceBus.AwsLambda.Tests +namespace NServiceBus.AcceptanceTests { using System.Collections.Generic; using Amazon.Lambda.SQSEvents; diff --git a/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/When_a_SQSEvent_with_large_payloads_is_processed.cs b/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/When_a_SQSEvent_with_large_payloads_is_processed.cs index 70a662f..92a8e78 100644 --- a/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/When_a_SQSEvent_with_large_payloads_is_processed.cs +++ b/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/When_a_SQSEvent_with_large_payloads_is_processed.cs @@ -1,4 +1,4 @@ -namespace NServiceBus.AwsLambda.Tests +namespace NServiceBus.AcceptanceTests { using System.Threading; using System.Threading.Tasks; @@ -19,10 +19,8 @@ public async Task The_handlers_should_be_invoked_and_process_successfully() var configuration = new AwsLambdaSQSEndpointConfiguration(QueueName, CreateSQSClient(), CreateSNSClient()); var transport = configuration.Transport; - transport.S3 = new S3Settings(BucketName, KeyPrefix, CreateS3Client()); - var advanced = configuration.AdvancedConfiguration; advanced.SendFailedMessagesTo(ErrorQueueName); advanced.RegisterComponents(c => c.AddSingleton(typeof(TestContext), context)); diff --git a/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/When_a_handler_sends_a_message.cs b/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/When_a_handler_sends_a_message.cs index d53b2a4..ba96bca 100644 --- a/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/When_a_handler_sends_a_message.cs +++ b/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/When_a_handler_sends_a_message.cs @@ -1,4 +1,4 @@ -namespace NServiceBus.AwsLambda.Tests +namespace NServiceBus.AcceptanceTests { using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; diff --git a/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/When_a_message_handler_always_throws.cs b/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/When_a_message_handler_always_throws.cs index 387dafe..68c775c 100644 --- a/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/When_a_message_handler_always_throws.cs +++ b/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/When_a_message_handler_always_throws.cs @@ -1,4 +1,4 @@ -namespace NServiceBus.AwsLambda.Tests +namespace NServiceBus.AcceptanceTests { using System; using System.Threading; diff --git a/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/When_the_SQS_trigger_is_activated.cs b/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/When_the_SQS_trigger_is_activated.cs index 94e4eab..2593ede 100644 --- a/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/When_the_SQS_trigger_is_activated.cs +++ b/src/NServiceBus.AwsLambda.SQS.AcceptanceTests/When_the_SQS_trigger_is_activated.cs @@ -1,4 +1,4 @@ -namespace NServiceBus.AwsLambda.Tests +namespace NServiceBus.AcceptanceTests { using System.Threading; using System.Threading.Tasks; From 4ce35dd4af1ed8b5ee774a203dbfb274c489292d Mon Sep 17 00:00:00 2001 From: danielmarbach Date: Tue, 4 Apr 2023 16:34:51 +0200 Subject: [PATCH 27/27] Remove condition --- .../TransportMessageExtensions.cs | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/NServiceBus.AwsLambda.SQS/TransportMessageExtensions.cs b/src/NServiceBus.AwsLambda.SQS/TransportMessageExtensions.cs index 567bbb5..5f033f1 100644 --- a/src/NServiceBus.AwsLambda.SQS/TransportMessageExtensions.cs +++ b/src/NServiceBus.AwsLambda.SQS/TransportMessageExtensions.cs @@ -58,16 +58,6 @@ static class TransportMessageExtensions } else { -#if NETFRAMEWORK - try - { - return (Convert.FromBase64String(body), null); - } - catch (FormatException) - { - return GetNonEncodedBody(body, arrayPool, null, encoding); - } -#else var buffer = GetBuffer(body, arrayPool, encoding); if (Convert.TryFromBase64String(body, buffer, out var writtenBytes)) { @@ -75,7 +65,6 @@ static class TransportMessageExtensions } return GetNonEncodedBody(body, arrayPool, buffer, encoding); -#endif } }