From 3a9a3286343cc2b22887fe172c30d90b94f3eb78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Ossowski?= Date: Sun, 5 Nov 2023 14:42:07 +0100 Subject: [PATCH 01/12] StartMessageMoveTask handling --- build.sbt | 2 +- .../elasticmq/actor/QueueManagerActor.scala | 30 ++++++ .../elasticmq/actor/queue/MessageQueue.scala | 17 ++++ .../actor/queue/QueueActorMessageOps.scala | 5 + .../operations/MoveMessagesAsyncOps.scala | 51 ++++++++++ .../org/elasticmq/msg/QueueManagerMsg.scala | 5 + .../scala/org/elasticmq/msg/QueueMsg.scala | 5 + .../scala/org/elasticmq/msg/package.scala | 5 + .../rest/sqs/AmazonJavaSdkTestSuite.scala | 1 - .../rest/sqs/MessageMoveTaskTest.scala | 79 +++++++++++++++ .../sqs/SqsClientServerCommunication.scala | 2 + .../scala/org/elasticmq/rest/sqs/Action.scala | 1 + .../rest/sqs/SQSRestServerBuilder.scala | 7 +- .../sqs/StartMessageMoveTaskDirectives.scala | 97 +++++++++++++++++++ 14 files changed, 304 insertions(+), 3 deletions(-) create mode 100644 core/src/main/scala/org/elasticmq/actor/queue/operations/MoveMessagesAsyncOps.scala create mode 100644 core/src/main/scala/org/elasticmq/msg/package.scala create mode 100644 rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/MessageMoveTaskTest.scala create mode 100644 rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/StartMessageMoveTaskDirectives.scala diff --git a/build.sbt b/build.sbt index 691c588bb..86a28e0fe 100644 --- a/build.sbt +++ b/build.sbt @@ -36,7 +36,7 @@ val jclOverSlf4j = "org.slf4j" % "jcl-over-slf4j" % "2.0.9" // needed form amazo val scalatest = "org.scalatest" %% "scalatest" % "3.2.17" val awaitility = "org.awaitility" % "awaitility-scala" % "4.2.0" -val amazonJavaSdkSqs = "com.amazonaws" % "aws-java-sdk-sqs" % "1.12.472" exclude ("commons-logging", "commons-logging") +val amazonJavaSdkSqs = "com.amazonaws" % "aws-java-sdk-sqs" % "1.12.580" exclude ("commons-logging", "commons-logging") val pekkoVersion = "1.0.1" val pekkoHttpVersion = "1.0.0" diff --git a/core/src/main/scala/org/elasticmq/actor/QueueManagerActor.scala b/core/src/main/scala/org/elasticmq/actor/QueueManagerActor.scala index 69d7ca300..242ecf835 100644 --- a/core/src/main/scala/org/elasticmq/actor/QueueManagerActor.scala +++ b/core/src/main/scala/org/elasticmq/actor/QueueManagerActor.scala @@ -1,13 +1,17 @@ package org.elasticmq.actor import org.apache.pekko.actor.{ActorRef, Props} +import org.apache.pekko.util.Timeout import org.elasticmq._ import org.elasticmq.actor.queue.{QueueActor, QueueEvent} import org.elasticmq.actor.reply._ import org.elasticmq.msg._ import org.elasticmq.util.{Logging, NowProvider} +import scala.concurrent.{ExecutionContext, Future} +import scala.concurrent.duration.DurationInt import scala.reflect._ +import scala.util.{Failure, Success} class QueueManagerActor(nowProvider: NowProvider, limits: Limits, queueEventListener: Option[ActorRef]) extends ReplyingActor @@ -15,6 +19,9 @@ class QueueManagerActor(nowProvider: NowProvider, limits: Limits, queueEventList type M[X] = QueueManagerMsg[X] val ev: ClassTag[QueueManagerMsg[Unit]] = classTag[M[Unit]] + implicit lazy val ec: ExecutionContext = context.dispatcher + implicit lazy val timeout: Timeout = 5.seconds + case class ActorWithQueueData(actorRef: ActorRef, queueData: QueueData) private val queues = collection.mutable.HashMap[String, ActorWithQueueData]() @@ -63,6 +70,29 @@ class QueueManagerActor(nowProvider: NowProvider, limits: Limits, queueEventList queues.collect { case (name, actor) if actor.queueData.deadLettersQueue.exists(_.name == queueName) => name }.toList + + case StartMessageMoveTask(sourceQueue, destinationQueue, maxNumberOfMessagesPerSecond) => + val replyTo = sender() + val destination = destinationQueue.map(Future.successful).getOrElse { + val queueData = sourceQueue ? GetQueueData() + queueData.map { qd => + queues + .filter { case (_, data) => + data.queueData.deadLettersQueue.exists(dlqd => dlqd.name == qd.name) + } + .head + ._2 + .actorRef + } + } + val f = destination.flatMap(destinationQueueActorRef => + sourceQueue ? StartMessageMoveTaskToQueue(destinationQueueActorRef, maxNumberOfMessagesPerSecond) + ) + f.onComplete { + case Success(value) => replyTo ! Right(value) + case Failure(ex) => logger.error("Failed to start message move task", ex) + } + DoNotReply() } protected def createQueueActor( diff --git a/core/src/main/scala/org/elasticmq/actor/queue/MessageQueue.scala b/core/src/main/scala/org/elasticmq/actor/queue/MessageQueue.scala index 79ec2f809..d1d560117 100644 --- a/core/src/main/scala/org/elasticmq/actor/queue/MessageQueue.scala +++ b/core/src/main/scala/org/elasticmq/actor/queue/MessageQueue.scala @@ -22,6 +22,13 @@ sealed trait MessageQueue { */ def getById(id: String): Option[InternalMessage] + /** Remove the first message from the queue + * + * @return + * The message or None if not exists + */ + def pop: Option[InternalMessage] + /** Get all messages in queue * * @return @@ -137,6 +144,16 @@ object MessageQueue { override def getById(id: String): Option[InternalMessage] = messagesById.get(id) + override def pop: Option[InternalMessage] = { + if (messageQueue.isEmpty) { + None + } else { + val firstMessage = messageQueue.dequeue() + remove(firstMessage.id) + Some(firstMessage) + } + } + override def all: Iterable[InternalMessage] = messagesById.values override def clear(): Unit = { diff --git a/core/src/main/scala/org/elasticmq/actor/queue/QueueActorMessageOps.scala b/core/src/main/scala/org/elasticmq/actor/queue/QueueActorMessageOps.scala index eccb62206..dff33e189 100644 --- a/core/src/main/scala/org/elasticmq/actor/queue/QueueActorMessageOps.scala +++ b/core/src/main/scala/org/elasticmq/actor/queue/QueueActorMessageOps.scala @@ -15,6 +15,7 @@ trait QueueActorMessageOps with DeleteMessageOps with ReceiveMessageOps with MoveMessageOps + with MoveMessagesAsyncOps with Timers { this: QueueActorStorage => @@ -39,6 +40,10 @@ trait QueueActorMessageOps fifoMessagesHistory = fifoMessagesHistory.cleanOutdatedMessages(nowProvider) DoNotReply() case RestoreMessages(messages) => restoreMessages(messages) + case StartMessageMoveTaskToQueue(destinationQueue, maxNumberOfMessagesPerSecond) => + startMovingMessages(destinationQueue, maxNumberOfMessagesPerSecond) + case MoveFirstMessageToQueue(destinationQueue, maxNumberOfMessagesPerSecond) => + moveFirstMessage(destinationQueue, maxNumberOfMessagesPerSecond).send() } } } diff --git a/core/src/main/scala/org/elasticmq/actor/queue/operations/MoveMessagesAsyncOps.scala b/core/src/main/scala/org/elasticmq/actor/queue/operations/MoveMessagesAsyncOps.scala new file mode 100644 index 000000000..b6f0c3e0f --- /dev/null +++ b/core/src/main/scala/org/elasticmq/actor/queue/operations/MoveMessagesAsyncOps.scala @@ -0,0 +1,51 @@ +package org.elasticmq.actor.queue.operations + +import org.apache.pekko.actor.ActorRef +import org.elasticmq.actor.queue.{InternalMessage, QueueActorStorage, QueueEvent} +import org.elasticmq.msg.{MoveFirstMessageToQueue, SendMessage, StartMessageMoveTaskId} +import org.elasticmq.util.Logging +import org.elasticmq.{DeduplicationId, MoveDestination, MoveToDLQ} + +import java.util.UUID +import scala.concurrent.duration.{DurationInt, FiniteDuration, NANOSECONDS} + +trait MoveMessagesAsyncOps extends Logging { + this: QueueActorStorage => + + def startMovingMessages( + destinationQueue: ActorRef, + maxNumberOfMessagesPerSecond: Option[Int] + ): StartMessageMoveTaskId = { + val taskId = UUID.randomUUID().toString + println("XXXX Start") + context.self ! MoveFirstMessageToQueue(destinationQueue, maxNumberOfMessagesPerSecond) + taskId + } + + def moveFirstMessage( + destinationQueue: ActorRef, + maxNumberOfMessagesPerSecond: Option[Int] + ): ResultWithEvents[Unit] = { + println("XXX " + messageQueue.all.toList.size) + messageQueue.pop match { + case Some(internalMessage) => + destinationQueue ! SendMessage(internalMessage.toNewMessageData) + maxNumberOfMessagesPerSecond match { + case Some(v) => + val nanosInSecond = 1.second.toNanos.toDouble + val delayNanos = (nanosInSecond / v).toLong + val delay = FiniteDuration(delayNanos, NANOSECONDS) + context.system.scheduler.scheduleOnce( + delay, + context.self, + MoveFirstMessageToQueue(destinationQueue, maxNumberOfMessagesPerSecond) + ) + case None => + context.self ! MoveFirstMessageToQueue(destinationQueue, maxNumberOfMessagesPerSecond) + } + ResultWithEvents.onlyEvents(List(QueueEvent.MessageRemoved(queueData.name, internalMessage.id))) + case None => + ResultWithEvents.empty + } + } +} diff --git a/core/src/main/scala/org/elasticmq/msg/QueueManagerMsg.scala b/core/src/main/scala/org/elasticmq/msg/QueueManagerMsg.scala index d11e5f371..c85002d96 100644 --- a/core/src/main/scala/org/elasticmq/msg/QueueManagerMsg.scala +++ b/core/src/main/scala/org/elasticmq/msg/QueueManagerMsg.scala @@ -11,3 +11,8 @@ case class DeleteQueue(queueName: String) extends QueueManagerMsg[Unit] case class LookupQueue(queueName: String) extends QueueManagerMsg[Option[ActorRef]] case class ListQueues() extends QueueManagerMsg[Seq[String]] case class ListDeadLetterSourceQueues(queueName: String) extends QueueManagerMsg[List[String]] +case class StartMessageMoveTask( + sourceQueue: ActorRef, + destinationQueue: Option[ActorRef], + maxNumberOfMessagesPerSecond: Option[Int] +) extends QueueManagerMsg[Either[ElasticMQError, StartMessageMoveTaskId]] diff --git a/core/src/main/scala/org/elasticmq/msg/QueueMsg.scala b/core/src/main/scala/org/elasticmq/msg/QueueMsg.scala index 7e28e3dad..fb48d5260 100644 --- a/core/src/main/scala/org/elasticmq/msg/QueueMsg.scala +++ b/core/src/main/scala/org/elasticmq/msg/QueueMsg.scala @@ -54,3 +54,8 @@ case class DeleteMessage(deliveryReceipt: DeliveryReceipt) extends QueueMessageM case class LookupMessage(messageId: MessageId) extends QueueMessageMsg[Option[MessageData]] case object DeduplicationIdsCleanup extends QueueMessageMsg[Unit] case class RestoreMessages(messages: List[InternalMessage]) extends QueueMessageMsg[Unit] +case class StartMessageMoveTaskToQueue(destinationQueue: ActorRef, maxNumberOfMessagesPerSecond: Option[Int]) + extends QueueMessageMsg[StartMessageMoveTaskId] + +case class MoveFirstMessageToQueue(destinationQueue: ActorRef, maxNumberOfMessagesPerSecond: Option[Int]) + extends QueueMessageMsg[Unit] diff --git a/core/src/main/scala/org/elasticmq/msg/package.scala b/core/src/main/scala/org/elasticmq/msg/package.scala new file mode 100644 index 000000000..894d7ec53 --- /dev/null +++ b/core/src/main/scala/org/elasticmq/msg/package.scala @@ -0,0 +1,5 @@ +package org.elasticmq + +package object msg { + type StartMessageMoveTaskId = String +} diff --git a/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/AmazonJavaSdkTestSuite.scala b/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/AmazonJavaSdkTestSuite.scala index 46947cf12..7b75746ed 100644 --- a/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/AmazonJavaSdkTestSuite.scala +++ b/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/AmazonJavaSdkTestSuite.scala @@ -24,7 +24,6 @@ import scala.util.control.Exception._ class AmazonJavaSdkTestSuite extends SqsClientServerCommunication with Matchers { val visibilityTimeoutAttribute = "VisibilityTimeout" val defaultVisibilityTimeoutAttribute = "VisibilityTimeout" - val redrivePolicyAttribute = "RedrivePolicy" val delaySecondsAttribute = "DelaySeconds" val receiveMessageWaitTimeSecondsAttribute = "ReceiveMessageWaitTimeSeconds" diff --git a/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/MessageMoveTaskTest.scala b/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/MessageMoveTaskTest.scala new file mode 100644 index 000000000..2aca544f1 --- /dev/null +++ b/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/MessageMoveTaskTest.scala @@ -0,0 +1,79 @@ +package org.elasticmq.rest.sqs + +import com.amazonaws.AmazonServiceException +import com.amazonaws.services.sqs.AmazonSQS +import com.amazonaws.services.sqs.model._ +import org.apache.http.HttpHost +import org.apache.http.client.entity.UrlEncodedFormEntity +import org.apache.http.client.methods.{HttpGet, HttpPost} +import org.apache.http.message.BasicNameValuePair +import org.apache.pekko.http.scaladsl.model.StatusCodes +import org.elasticmq._ +import org.elasticmq.rest.sqs.model.RedrivePolicy +import org.elasticmq.rest.sqs.model.RedrivePolicyJson.format +import org.scalatest.matchers.should.Matchers +import spray.json.enrichAny + +import java.net.URI +import java.nio.ByteBuffer +import java.util.UUID +import scala.collection.JavaConverters._ +import scala.io.Source +import scala.util.control.Exception._ + +class MessageMoveTaskTest extends SqsClientServerCommunication with Matchers { + + test("blah blah blah") { + val dlq = client.createQueue(new CreateQueueRequest("testQueue-dlq")) + val redrivePolicy = RedrivePolicy("testQueue-dlq", awsRegion, awsAccountId, 1).toJson.toString() + val q = + client.createQueue( + new CreateQueueRequest("testQueue") + .addAttributesEntry(redrivePolicyAttribute, redrivePolicy) + .addAttributesEntry("VisibilityTimeout", "1") + ) + + val volume = 6 + for (i <- 0 until volume) { + client.sendMessage(q.getQueueUrl, "Test message " + i) + } + + for (i <- 0 until volume) { + client.receiveMessage(q.getQueueUrl) + } + + Thread.sleep(2000) + for (i <- 0 until volume) { + client.receiveMessage(q.getQueueUrl) + } + + Thread.sleep(2000) + println( + client.getQueueAttributes( + new GetQueueAttributesRequest().withQueueUrl(q.getQueueUrl).withAttributeNames("ApproximateNumberOfMessages") + ) + ) + println( + client.getQueueAttributes( + new GetQueueAttributesRequest().withQueueUrl(dlq.getQueueUrl).withAttributeNames("ApproximateNumberOfMessages") + ) + ) + + val response = client.startMessageMoveTask( + new StartMessageMoveTaskRequest().withSourceArn(s"arn:aws:sqs:$awsRegion:$awsAccountId:testQueue-dlq") + ) + println(response.getTaskHandle) + + Thread.sleep(5000) + println( + client.getQueueAttributes( + new GetQueueAttributesRequest().withQueueUrl(q.getQueueUrl).withAttributeNames("ApproximateNumberOfMessages") + ) + ) + println( + client.getQueueAttributes( + new GetQueueAttributesRequest().withQueueUrl(dlq.getQueueUrl).withAttributeNames("ApproximateNumberOfMessages") + ) + ) + } +} diff --git a/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/SqsClientServerCommunication.scala b/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/SqsClientServerCommunication.scala index ae094cd89..423bfe1a0 100644 --- a/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/SqsClientServerCommunication.scala +++ b/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/SqsClientServerCommunication.scala @@ -28,6 +28,8 @@ trait SqsClientServerCommunication extends AnyFunSuite with BeforeAndAfter with val ServiceEndpoint = "http://localhost:9321" + val redrivePolicyAttribute = "RedrivePolicy" + before { logger.info(s"\n---\nRunning test: $currentTestName\n---\n") diff --git a/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/Action.scala b/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/Action.scala index c2570c360..de793b676 100644 --- a/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/Action.scala +++ b/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/Action.scala @@ -21,4 +21,5 @@ object Action extends Enumeration { val SendMessage = Value("SendMessage") val TagQueue = Value("TagQueue") val UntagQueue = Value("UntagQueue") + val StartMessageMoveTask = Value("StartMessageMoveTask") } diff --git a/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/SQSRestServerBuilder.scala b/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/SQSRestServerBuilder.scala index 0d117d605..c3a34d7ac 100644 --- a/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/SQSRestServerBuilder.scala +++ b/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/SQSRestServerBuilder.scala @@ -170,7 +170,8 @@ case class TheSQSRestServerBuilder( with UnmatchedActionRoutes with ResponseMarshaller with QueueAttributesOps - with ListDeadLetterSourceQueuesDirectives { + with ListDeadLetterSourceQueuesDirectives + with StartMessageMoveTaskDirectives { def serverAddress = currentServerAddress.get() @@ -211,6 +212,7 @@ case class TheSQSRestServerBuilder( untagQueue(p) ~ listQueueTags(p) ~ listDeadLetterSourceQueues(p) ~ + startMessageMoveTask(p) ~ // 4. Unmatched action unmatchedAction(p) @@ -333,6 +335,9 @@ object Constants { val MaxResultsParameter = "MaxResults" val NextTokenParameter = "NextToken" val QueueNamePrefixParameter = "QueueNamePrefix" + val SourceArnParameter = "SourceArn" + val DestinationArnParameter = "DestinationArn" + val MaxNumberOfMessagesPerSecondParameter = "MaxNumberOfMessagesPerSecond" } object ParametersUtil { diff --git a/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/StartMessageMoveTaskDirectives.scala b/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/StartMessageMoveTaskDirectives.scala new file mode 100644 index 000000000..206152dd7 --- /dev/null +++ b/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/StartMessageMoveTaskDirectives.scala @@ -0,0 +1,97 @@ +package org.elasticmq.rest.sqs + +import Constants._ +import org.apache.pekko.actor.ActorRef +import org.apache.pekko.http.scaladsl.server.{Directive1, Route} +import org.elasticmq.ElasticMQError +import org.elasticmq.actor.reply._ + +import scala.async.Async._ +import org.elasticmq.msg.StartMessageMoveTask +import org.elasticmq.rest.sqs.Action.{StartMessageMoveTask => StartMessageMoveTaskAction} +import org.elasticmq.rest.sqs.directives.ElasticMQDirectives +import org.elasticmq.rest.sqs.model.RequestPayload +import spray.json.DefaultJsonProtocol._ +import spray.json.RootJsonFormat + +trait StartMessageMoveTaskDirectives { this: ElasticMQDirectives with QueueURLModule with ResponseMarshaller => + + private val Arn = "(?:.+:(.+)?:(.+)?:)?(.+)".r + + def startMessageMoveTask(p: RequestPayload)(implicit marshallerDependencies: MarshallerDependencies): Route = { + p.action(StartMessageMoveTaskAction) { + val params = p.as[StartMessageMoveTaskActionRequest] + val sourceQueueName = arnToQueueName(params.SourceArn) + queueActorAndDataFromQueueName(sourceQueueName) { (sourceQueue, _) => + params.DestinationArn match { + case Some(destinationQueueArn) => + val destinationQueueName = arnToQueueName(destinationQueueArn) + queueActorAndDataFromQueueName(destinationQueueName) { (destinationQueue, _) => + startMessageMoveTask(sourceQueue, Some(destinationQueue), params.MaxNumberOfMessagesPerSecond) + } + case None => startMessageMoveTask(sourceQueue, None, params.MaxNumberOfMessagesPerSecond) + } + } + } + } + + private def arnToQueueName(arn: String): String = + arn match { + case Arn(_, _, queueName) => queueName + case _ => throw new SQSException("InvalidParameterValue") + } + + private def startMessageMoveTask( + sourceQueue: ActorRef, + destinationQueue: Option[ActorRef], + maxNumberOfMessagesPerSecond: Option[Int] + )(implicit marshallerDependencies: MarshallerDependencies): Route = + async { + await( + queueManagerActor ? StartMessageMoveTask(sourceQueue, destinationQueue, maxNumberOfMessagesPerSecond) + ) match { + case Left(e: ElasticMQError) => throw new SQSException(e.code, errorMessage = Some(e.message)) + case Right(taskHandle) => + println("XXX" + taskHandle) + complete(StartMessageMoveTaskResponse(taskHandle)) + } + } +} + +case class StartMessageMoveTaskActionRequest( + SourceArn: String, + DestinationArn: Option[String], + MaxNumberOfMessagesPerSecond: Option[Int] +) + +object StartMessageMoveTaskActionRequest { + implicit val requestJsonFormat: RootJsonFormat[StartMessageMoveTaskActionRequest] = jsonFormat3( + StartMessageMoveTaskActionRequest.apply + ) + + implicit val requestParamReader: FlatParamsReader[StartMessageMoveTaskActionRequest] = + new FlatParamsReader[StartMessageMoveTaskActionRequest] { + override def read(params: Map[String, String]): StartMessageMoveTaskActionRequest = { + new StartMessageMoveTaskActionRequest( + requiredParameter(params)(SourceArnParameter), + optionalParameter(params)(DestinationArnParameter), + optionalParameter(params)(MaxNumberOfMessagesPerSecondParameter).map(_.toInt) + ) + } + } +} + +case class StartMessageMoveTaskResponse(TaskHandle: String) + +object StartMessageMoveTaskResponse { + implicit val format: RootJsonFormat[StartMessageMoveTaskResponse] = jsonFormat1(StartMessageMoveTaskResponse.apply) + + implicit val xmlSerializer: XmlSerializer[StartMessageMoveTaskResponse] = t => + + {t.TaskHandle} + + + {EmptyRequestId} + + +} From a1f42300a782e6fc8a21caf494c26f3a9238fa7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Ossowski?= Date: Thu, 4 Apr 2024 08:52:48 +0200 Subject: [PATCH 02/12] add some logging, test cases --- .../elasticmq/actor/QueueManagerActor.scala | 23 +-- .../elasticmq/actor/queue/MessageQueue.scala | 4 + .../elasticmq/actor/queue/QueueActor.scala | 6 +- .../actor/queue/QueueActorStorage.scala | 4 +- .../operations/MoveMessagesAsyncOps.scala | 5 +- .../rest/sqs/MessageMoveTaskTest.scala | 170 ++++++++++++++---- 6 files changed, 164 insertions(+), 48 deletions(-) diff --git a/core/src/main/scala/org/elasticmq/actor/QueueManagerActor.scala b/core/src/main/scala/org/elasticmq/actor/QueueManagerActor.scala index 242ecf835..be3fc3ad8 100644 --- a/core/src/main/scala/org/elasticmq/actor/QueueManagerActor.scala +++ b/core/src/main/scala/org/elasticmq/actor/QueueManagerActor.scala @@ -74,23 +74,25 @@ class QueueManagerActor(nowProvider: NowProvider, limits: Limits, queueEventList case StartMessageMoveTask(sourceQueue, destinationQueue, maxNumberOfMessagesPerSecond) => val replyTo = sender() val destination = destinationQueue.map(Future.successful).getOrElse { - val queueData = sourceQueue ? GetQueueData() - queueData.map { qd => + val queueDataF = sourceQueue ? GetQueueData() + queueDataF.map { queueData => queues .filter { case (_, data) => - data.queueData.deadLettersQueue.exists(dlqd => dlqd.name == qd.name) + data.queueData.deadLettersQueue.exists(dlqData => dlqData.name == queueData.name) } .head ._2 .actorRef } } - val f = destination.flatMap(destinationQueueActorRef => - sourceQueue ? StartMessageMoveTaskToQueue(destinationQueueActorRef, maxNumberOfMessagesPerSecond) - ) - f.onComplete { - case Success(value) => replyTo ! Right(value) - case Failure(ex) => logger.error("Failed to start message move task", ex) + destination.flatMap(destinationQueueActorRef => { + val taskIdF = sourceQueue ? StartMessageMoveTaskToQueue(destinationQueueActorRef, maxNumberOfMessagesPerSecond) + taskIdF.map(taskId => (taskId, destinationQueueActorRef)) + }).onComplete { + case Success((taskId, destinationQueueActorRef)) => + logger.debug("Message move task {} => {} created", sourceQueue, destinationQueueActorRef) + replyTo ! Right(taskId) + case Failure(ex) => logger.error("Failed to start message move task", ex) } DoNotReply() } @@ -118,7 +120,8 @@ class QueueManagerActor(nowProvider: NowProvider, limits: Limits, queueEventList moveMessagesToQueueActor, queueEventListener ) - ) + ), + s"queue-${queueData.name}" ) } diff --git a/core/src/main/scala/org/elasticmq/actor/queue/MessageQueue.scala b/core/src/main/scala/org/elasticmq/actor/queue/MessageQueue.scala index d1d560117..7e6c50331 100644 --- a/core/src/main/scala/org/elasticmq/actor/queue/MessageQueue.scala +++ b/core/src/main/scala/org/elasticmq/actor/queue/MessageQueue.scala @@ -36,6 +36,8 @@ sealed trait MessageQueue { */ def all: Iterable[InternalMessage] + def size: Long + /** Drop all messages on the queue */ def clear(): Unit @@ -156,6 +158,8 @@ object MessageQueue { override def all: Iterable[InternalMessage] = messagesById.values + override def size: Long = messageQueue.size + override def clear(): Unit = { messagesById.clear() messageQueue.clear() diff --git a/core/src/main/scala/org/elasticmq/actor/queue/QueueActor.scala b/core/src/main/scala/org/elasticmq/actor/queue/QueueActor.scala index e81d1a222..ae9f72632 100644 --- a/core/src/main/scala/org/elasticmq/actor/queue/QueueActor.scala +++ b/core/src/main/scala/org/elasticmq/actor/queue/QueueActor.scala @@ -1,8 +1,8 @@ package org.elasticmq.actor.queue -import org.apache.pekko.actor.ActorRef +import org.apache.pekko.actor.{ActorLogging, ActorRef} import org.elasticmq.QueueData -import org.elasticmq.actor.reply.ReplyingActor +import org.elasticmq.actor.reply.{ReplyAction, ReplyingActor} import org.elasticmq.msg._ import org.elasticmq.util.{Logging, NowProvider} @@ -24,7 +24,7 @@ class QueueActor( type M[X] = QueueMsg[X] val ev = classTag[M[Unit]] - def receiveAndReply[T](msg: QueueMsg[T]) = + def receiveAndReply[T](msg: QueueMsg[T]): ReplyAction[T] = msg match { case m: QueueQueueMsg[T] => val replyAction = receiveAndReplyQueueMsg(m) diff --git a/core/src/main/scala/org/elasticmq/actor/queue/QueueActorStorage.scala b/core/src/main/scala/org/elasticmq/actor/queue/QueueActorStorage.scala index f021ab15e..24df654e9 100644 --- a/core/src/main/scala/org/elasticmq/actor/queue/QueueActorStorage.scala +++ b/core/src/main/scala/org/elasticmq/actor/queue/QueueActorStorage.scala @@ -55,7 +55,9 @@ trait QueueActorStorage { notificationF.onComplete { case Success(_) => result match { - case Some(r) => actualSender ! r + case Some(r) => + logger.debug(s"Sending message $r from ${context.self} to $actualSender") + actualSender ! r case None => } case Failure(ex) => logger.error(s"Failed to notify queue event listener. The state may be inconsistent.", ex) diff --git a/core/src/main/scala/org/elasticmq/actor/queue/operations/MoveMessagesAsyncOps.scala b/core/src/main/scala/org/elasticmq/actor/queue/operations/MoveMessagesAsyncOps.scala index b6f0c3e0f..5f7565561 100644 --- a/core/src/main/scala/org/elasticmq/actor/queue/operations/MoveMessagesAsyncOps.scala +++ b/core/src/main/scala/org/elasticmq/actor/queue/operations/MoveMessagesAsyncOps.scala @@ -1,6 +1,7 @@ package org.elasticmq.actor.queue.operations import org.apache.pekko.actor.ActorRef +import org.apache.pekko.pattern.ask import org.elasticmq.actor.queue.{InternalMessage, QueueActorStorage, QueueEvent} import org.elasticmq.msg.{MoveFirstMessageToQueue, SendMessage, StartMessageMoveTaskId} import org.elasticmq.util.Logging @@ -17,7 +18,7 @@ trait MoveMessagesAsyncOps extends Logging { maxNumberOfMessagesPerSecond: Option[Int] ): StartMessageMoveTaskId = { val taskId = UUID.randomUUID().toString - println("XXXX Start") + logger.debug("Starting message move task to queue {} (task id: {})", destinationQueue, taskId) context.self ! MoveFirstMessageToQueue(destinationQueue, maxNumberOfMessagesPerSecond) taskId } @@ -26,7 +27,7 @@ trait MoveMessagesAsyncOps extends Logging { destinationQueue: ActorRef, maxNumberOfMessagesPerSecond: Option[Int] ): ResultWithEvents[Unit] = { - println("XXX " + messageQueue.all.toList.size) + logger.debug("Trying to move a single message to {} ({} messages left)", destinationQueue, messageQueue.size) messageQueue.pop match { case Some(internalMessage) => destinationQueue ! SendMessage(internalMessage.toNewMessageData) diff --git a/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/MessageMoveTaskTest.scala b/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/MessageMoveTaskTest.scala index 2aca544f1..bd093eb00 100644 --- a/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/MessageMoveTaskTest.scala +++ b/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/MessageMoveTaskTest.scala @@ -2,6 +2,7 @@ package org.elasticmq.rest.sqs import com.amazonaws.AmazonServiceException import com.amazonaws.services.sqs.AmazonSQS +import com.amazonaws.services.sqs.model.QueueAttributeName.ApproximateNumberOfMessages import com.amazonaws.services.sqs.model._ import org.apache.http.HttpHost import org.apache.http.client.entity.UrlEncodedFormEntity @@ -11,6 +12,7 @@ import org.apache.pekko.http.scaladsl.model.StatusCodes import org.elasticmq._ import org.elasticmq.rest.sqs.model.RedrivePolicy import org.elasticmq.rest.sqs.model.RedrivePolicyJson.format +import org.scalatest.concurrent.Eventually import org.scalatest.matchers.should.Matchers import spray.json.enrichAny @@ -18,62 +20,166 @@ import java.net.URI import java.nio.ByteBuffer import java.util.UUID import scala.collection.JavaConverters._ +import scala.concurrent.duration.DurationInt import scala.io.Source import scala.util.control.Exception._ -class MessageMoveTaskTest extends SqsClientServerCommunication with Matchers { +class MessageMoveTaskTest extends SqsClientServerCommunication with Matchers with Eventually { - test("blah blah blah") { + test("should run message move task") { + // given val dlq = client.createQueue(new CreateQueueRequest("testQueue-dlq")) val redrivePolicy = RedrivePolicy("testQueue-dlq", awsRegion, awsAccountId, 1).toJson.toString() - val q = + val queue = client.createQueue( new CreateQueueRequest("testQueue") .addAttributesEntry(redrivePolicyAttribute, redrivePolicy) .addAttributesEntry("VisibilityTimeout", "1") ) - val volume = 6 - for (i <- 0 until volume) { - client.sendMessage(q.getQueueUrl, "Test message " + i) + // when: send messages + val numMessages = 6 + for (i <- 0 until numMessages) { + client.sendMessage(queue.getQueueUrl, "Test message " + i) } - for (i <- 0 until volume) { - client.receiveMessage(q.getQueueUrl) + // and: receive messages + for (i <- 0 until numMessages) { + client.receiveMessage(queue.getQueueUrl) } - Thread.sleep(2000) - for (i <- 0 until volume) { - client.receiveMessage(q.getQueueUrl) + // and: receive messages again to make them move to DLQ + Thread.sleep(1500) + for (i <- 0 until numMessages) { + client.receiveMessage(queue.getQueueUrl) } - Thread.sleep(2000) - println( - client.getQueueAttributes( - new GetQueueAttributesRequest().withQueueUrl(q.getQueueUrl).withAttributeNames("ApproximateNumberOfMessages") - ) - ) - println( - client.getQueueAttributes( - new GetQueueAttributesRequest().withQueueUrl(dlq.getQueueUrl).withAttributeNames("ApproximateNumberOfMessages") - ) - ) + // then: ensure that messages are in DLQ + eventually(timeout(2.seconds), interval(100.millis)) { + fetchApproximateNumberOfMessages(queue.getQueueUrl) shouldEqual 0 + fetchApproximateNumberOfMessages(dlq.getQueueUrl) shouldEqual numMessages + } - val response = client.startMessageMoveTask( + // when: start message move task + client.startMessageMoveTask( new StartMessageMoveTaskRequest().withSourceArn(s"arn:aws:sqs:$awsRegion:$awsAccountId:testQueue-dlq") ) - println(response.getTaskHandle) - Thread.sleep(5000) - println( - client.getQueueAttributes( - new GetQueueAttributesRequest().withQueueUrl(q.getQueueUrl).withAttributeNames("ApproximateNumberOfMessages") + // then: ensure that messages are moved back to the original queue + eventually(timeout(5.seconds), interval(100.millis)) { + fetchApproximateNumberOfMessages(queue.getQueueUrl) shouldEqual numMessages + fetchApproximateNumberOfMessages(dlq.getQueueUrl) shouldEqual 0 + } + } + + test("should run message move task with max number of messages per second") { + // given + val dlq = client.createQueue(new CreateQueueRequest("testQueue-dlq")) + val redrivePolicy = RedrivePolicy("testQueue-dlq", awsRegion, awsAccountId, 1).toJson.toString() + val queue = + client.createQueue( + new CreateQueueRequest("testQueue") + .addAttributesEntry(redrivePolicyAttribute, redrivePolicy) + .addAttributesEntry("VisibilityTimeout", "1") ) + + // when: send messages + val numMessages = 6 + for (i <- 0 until numMessages) { + client.sendMessage(queue.getQueueUrl, "Test message " + i) + } + + // and: receive messages + for (i <- 0 until numMessages) { + client.receiveMessage(queue.getQueueUrl) + } + + // and: receive messages again to make them move to DLQ + Thread.sleep(1500) + for (i <- 0 until numMessages) { + client.receiveMessage(queue.getQueueUrl) + } + + // then: ensure that messages are in DLQ + eventually(timeout(2.seconds), interval(100.millis)) { + fetchApproximateNumberOfMessages(queue.getQueueUrl) shouldEqual 0 + fetchApproximateNumberOfMessages(dlq.getQueueUrl) shouldEqual numMessages + } + + // when: start message move task + client.startMessageMoveTask( + new StartMessageMoveTaskRequest() + .withSourceArn(s"arn:aws:sqs:$awsRegion:$awsAccountId:testQueue-dlq") + .withMaxNumberOfMessagesPerSecond(1) ) - println( - client.getQueueAttributes( - new GetQueueAttributesRequest().withQueueUrl(dlq.getQueueUrl).withAttributeNames("ApproximateNumberOfMessages") + + // then: ensure that not all messages were moved back to the original queue after 2 seconds + Thread.sleep(2000) + fetchApproximateNumberOfMessages(queue.getQueueUrl) should (be > 1 and be < 6) + fetchApproximateNumberOfMessages(dlq.getQueueUrl) should (be > 1 and be < 6) + } + + test("should run and cancel message move task") { + // given + val dlq = client.createQueue(new CreateQueueRequest("testQueue-dlq")) + val redrivePolicy = RedrivePolicy("testQueue-dlq", awsRegion, awsAccountId, 1).toJson.toString() + val queue = + client.createQueue( + new CreateQueueRequest("testQueue") + .addAttributesEntry(redrivePolicyAttribute, redrivePolicy) + .addAttributesEntry("VisibilityTimeout", "1") ) - ) + + // when: send messages + val numMessages = 6 + for (i <- 0 until numMessages) { + client.sendMessage(queue.getQueueUrl, "Test message " + i) + } + + // and: receive messages + for (i <- 0 until numMessages) { + client.receiveMessage(queue.getQueueUrl) + } + + // and: receive messages again to make them move to DLQ + Thread.sleep(1500) + for (i <- 0 until numMessages) { + client.receiveMessage(queue.getQueueUrl) + } + + // then: ensure that messages are in DLQ + eventually(timeout(2.seconds), interval(100.millis)) { + fetchApproximateNumberOfMessages(queue.getQueueUrl) shouldEqual 0 + fetchApproximateNumberOfMessages(dlq.getQueueUrl) shouldEqual numMessages + } + + // when: start message move task + val taskHandle = client.startMessageMoveTask( + new StartMessageMoveTaskRequest() + .withSourceArn(s"arn:aws:sqs:$awsRegion:$awsAccountId:testQueue-dlq") + .withMaxNumberOfMessagesPerSecond(1) + ).getTaskHandle + + // and: cancel the task after 2 seconds + Thread.sleep(2000) + client.cancelMessageMoveTask(new CancelMessageMoveTaskRequest().withTaskHandle(taskHandle)) + + // and: fetch ApproximateNumberOfMessages + val numMessagesInMainQueue = fetchApproximateNumberOfMessages(queue.getQueueUrl) + val numMessagesInDlQueue = fetchApproximateNumberOfMessages(dlq.getQueueUrl) + + // then: ApproximateNumberOfMessages should not change after 2 seconds + Thread.sleep(2000) + fetchApproximateNumberOfMessages(queue.getQueueUrl) shouldEqual numMessagesInMainQueue + fetchApproximateNumberOfMessages(dlq.getQueueUrl) shouldEqual numMessagesInDlQueue + } + + private def fetchApproximateNumberOfMessages(queueUrl: String): Int = { + client + .getQueueAttributes( + new GetQueueAttributesRequest().withQueueUrl(queueUrl).withAttributeNames(ApproximateNumberOfMessages) + ) + .getAttributes + .get(ApproximateNumberOfMessages.toString).toInt } } From c49c57e724ee142667fc7d06b59ebc28040ba2d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Ossowski?= Date: Thu, 4 Apr 2024 10:21:50 +0200 Subject: [PATCH 03/12] implement CancelMessageMoveTask --- .../scala/org/elasticmq/ElasticMQError.scala | 8 ++ .../elasticmq/actor/QueueManagerActor.scala | 45 ++++++++--- .../actor/queue/QueueActorMessageOps.scala | 10 ++- .../operations/MoveMessagesAsyncOps.scala | 79 +++++++++++++------ .../org/elasticmq/msg/QueueManagerMsg.scala | 8 +- .../scala/org/elasticmq/msg/QueueMsg.scala | 14 ++-- .../scala/org/elasticmq/msg/package.scala | 2 +- .../scala/org/elasticmq/rest/sqs/Action.scala | 1 + .../sqs/CancelMessageMoveTaskDirectives.scala | 66 ++++++++++++++++ .../rest/sqs/SQSRestServerBuilder.scala | 15 ++-- .../sqs/StartMessageMoveTaskDirectives.scala | 12 ++- 11 files changed, 196 insertions(+), 64 deletions(-) create mode 100644 rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/CancelMessageMoveTaskDirectives.scala diff --git a/core/src/main/scala/org/elasticmq/ElasticMQError.scala b/core/src/main/scala/org/elasticmq/ElasticMQError.scala index 47571ffe8..7a70bc442 100644 --- a/core/src/main/scala/org/elasticmq/ElasticMQError.scala +++ b/core/src/main/scala/org/elasticmq/ElasticMQError.scala @@ -1,4 +1,5 @@ package org.elasticmq +import org.elasticmq.msg.MessageMoveTaskId trait ElasticMQError { val queueName: String @@ -30,3 +31,10 @@ class InvalidReceiptHandle(val queueName: String, receiptHandle: String) extends val code = "ReceiptHandleIsInvalid" val message = s"""The receipt handle "$receiptHandle" is not valid.""" } + +class InvalidMessageMoveTaskId(val taskId: MessageMoveTaskId) extends ElasticMQError { + val code = "InvalidMessageMoveTaskId" + val message = s"""The task id "$taskId" is not valid or does not exist""" + + override val queueName: String = "invalid" +} diff --git a/core/src/main/scala/org/elasticmq/actor/QueueManagerActor.scala b/core/src/main/scala/org/elasticmq/actor/QueueManagerActor.scala index be3fc3ad8..d760c8783 100644 --- a/core/src/main/scala/org/elasticmq/actor/QueueManagerActor.scala +++ b/core/src/main/scala/org/elasticmq/actor/QueueManagerActor.scala @@ -8,8 +8,8 @@ import org.elasticmq.actor.reply._ import org.elasticmq.msg._ import org.elasticmq.util.{Logging, NowProvider} -import scala.concurrent.{ExecutionContext, Future} import scala.concurrent.duration.DurationInt +import scala.concurrent.{ExecutionContext, Future} import scala.reflect._ import scala.util.{Failure, Success} @@ -24,8 +24,10 @@ class QueueManagerActor(nowProvider: NowProvider, limits: Limits, queueEventList case class ActorWithQueueData(actorRef: ActorRef, queueData: QueueData) private val queues = collection.mutable.HashMap[String, ActorWithQueueData]() + private val messageMoveTasks = collection.mutable.HashMap[MessageMoveTaskId, ActorRef]() - def receiveAndReply[T](msg: QueueManagerMsg[T]): ReplyAction[T] = + def receiveAndReply[T](msg: QueueManagerMsg[T]): ReplyAction[T] = { + val self = context.self msg match { case CreateQueue(request) => queues.get(request.name) match { @@ -85,17 +87,38 @@ class QueueManagerActor(nowProvider: NowProvider, limits: Limits, queueEventList .actorRef } } - destination.flatMap(destinationQueueActorRef => { - val taskIdF = sourceQueue ? StartMessageMoveTaskToQueue(destinationQueueActorRef, maxNumberOfMessagesPerSecond) - taskIdF.map(taskId => (taskId, destinationQueueActorRef)) - }).onComplete { - case Success((taskId, destinationQueueActorRef)) => - logger.debug("Message move task {} => {} created", sourceQueue, destinationQueueActorRef) - replyTo ! Right(taskId) - case Failure(ex) => logger.error("Failed to start message move task", ex) - } + destination + .flatMap(destinationQueueActorRef => { + val taskIdF = + sourceQueue ? StartMovingMessages(destinationQueueActorRef, maxNumberOfMessagesPerSecond, self) + taskIdF.map(taskId => (taskId, destinationQueueActorRef)) + }) + .onComplete { + case Success((taskId, destinationQueueActorRef)) => + logger.debug("Message move task {} => {} created", sourceQueue, destinationQueueActorRef) + messageMoveTasks.put(taskId, sourceQueue) + replyTo ! Right(taskId) + case Failure(ex) => logger.error("Failed to start message move task", ex) + } + DoNotReply() + + case MessageMoveTaskFinished(taskHandle) => + logger.debug("Message move task {} finished", taskHandle) + messageMoveTasks.remove(taskHandle) DoNotReply() + + case CancelMessageMoveTask(taskHandle) => + logger.info("Cancelling message move task {}", taskHandle) + messageMoveTasks.get(taskHandle) match { + case Some(sourceQueue) => + sourceQueue ! CancelMovingMessages() + messageMoveTasks.remove(taskHandle) + ReplyWith(Right(0)) + case None => + ReplyWith(Left(new InvalidMessageMoveTaskId(taskHandle))) + } } + } protected def createQueueActor( nowProvider: NowProvider, diff --git a/core/src/main/scala/org/elasticmq/actor/queue/QueueActorMessageOps.scala b/core/src/main/scala/org/elasticmq/actor/queue/QueueActorMessageOps.scala index 8e8ed2aa5..17bd05b44 100644 --- a/core/src/main/scala/org/elasticmq/actor/queue/QueueActorMessageOps.scala +++ b/core/src/main/scala/org/elasticmq/actor/queue/QueueActorMessageOps.scala @@ -39,10 +39,12 @@ trait QueueActorMessageOps fifoMessagesHistory = fifoMessagesHistory.cleanOutdatedMessages(nowProvider) DoNotReply() case RestoreMessages(messages) => restoreMessages(messages) - case StartMessageMoveTaskToQueue(destinationQueue, maxNumberOfMessagesPerSecond) => - startMovingMessages(destinationQueue, maxNumberOfMessagesPerSecond) - case MoveFirstMessageToQueue(destinationQueue, maxNumberOfMessagesPerSecond) => - moveFirstMessage(destinationQueue, maxNumberOfMessagesPerSecond).send() + case StartMovingMessages(destinationQueue, maxNumberOfMessagesPerSecond, queueManager) => + startMovingMessages(destinationQueue, maxNumberOfMessagesPerSecond, queueManager) + case CancelMovingMessages() => + cancelMovingMessages() + case MoveFirstMessage(taskId, destinationQueue, maxNumberOfMessagesPerSecond, queueManager) => + moveFirstMessage(taskId, destinationQueue, maxNumberOfMessagesPerSecond, queueManager).send() } } } diff --git a/core/src/main/scala/org/elasticmq/actor/queue/operations/MoveMessagesAsyncOps.scala b/core/src/main/scala/org/elasticmq/actor/queue/operations/MoveMessagesAsyncOps.scala index 5f7565561..a4fb8f3ba 100644 --- a/core/src/main/scala/org/elasticmq/actor/queue/operations/MoveMessagesAsyncOps.scala +++ b/core/src/main/scala/org/elasticmq/actor/queue/operations/MoveMessagesAsyncOps.scala @@ -1,52 +1,79 @@ package org.elasticmq.actor.queue.operations import org.apache.pekko.actor.ActorRef -import org.apache.pekko.pattern.ask -import org.elasticmq.actor.queue.{InternalMessage, QueueActorStorage, QueueEvent} -import org.elasticmq.msg.{MoveFirstMessageToQueue, SendMessage, StartMessageMoveTaskId} +import org.elasticmq.actor.queue.{QueueActorStorage, QueueEvent} +import org.elasticmq.msg.{MessageMoveTaskFinished, MessageMoveTaskId, MoveFirstMessage, SendMessage} import org.elasticmq.util.Logging -import org.elasticmq.{DeduplicationId, MoveDestination, MoveToDLQ} import java.util.UUID import scala.concurrent.duration.{DurationInt, FiniteDuration, NANOSECONDS} +sealed trait MessageMoveTaskState +case object NotMovingMessages extends MessageMoveTaskState +case class MovingMessagesInProgress(numMessagesMoved: Int) extends MessageMoveTaskState + trait MoveMessagesAsyncOps extends Logging { this: QueueActorStorage => + private var messageMoveTaskState: MessageMoveTaskState = NotMovingMessages + def startMovingMessages( destinationQueue: ActorRef, - maxNumberOfMessagesPerSecond: Option[Int] - ): StartMessageMoveTaskId = { + maxNumberOfMessagesPerSecond: Option[Int], + queueManager: ActorRef + ): MessageMoveTaskId = { val taskId = UUID.randomUUID().toString logger.debug("Starting message move task to queue {} (task id: {})", destinationQueue, taskId) - context.self ! MoveFirstMessageToQueue(destinationQueue, maxNumberOfMessagesPerSecond) + messageMoveTaskState = MovingMessagesInProgress(0) + context.self ! MoveFirstMessage(taskId, destinationQueue, maxNumberOfMessagesPerSecond, queueManager) taskId } def moveFirstMessage( + taskId: MessageMoveTaskId, destinationQueue: ActorRef, - maxNumberOfMessagesPerSecond: Option[Int] + maxNumberOfMessagesPerSecond: Option[Int], + queueManager: ActorRef ): ResultWithEvents[Unit] = { - logger.debug("Trying to move a single message to {} ({} messages left)", destinationQueue, messageQueue.size) - messageQueue.pop match { - case Some(internalMessage) => - destinationQueue ! SendMessage(internalMessage.toNewMessageData) - maxNumberOfMessagesPerSecond match { - case Some(v) => - val nanosInSecond = 1.second.toNanos.toDouble - val delayNanos = (nanosInSecond / v).toLong - val delay = FiniteDuration(delayNanos, NANOSECONDS) - context.system.scheduler.scheduleOnce( - delay, - context.self, - MoveFirstMessageToQueue(destinationQueue, maxNumberOfMessagesPerSecond) - ) + messageMoveTaskState match { + case NotMovingMessages => + logger.debug("Moving messages task {} was finished or cancelled", taskId) + ResultWithEvents.empty + case MovingMessagesInProgress(numMessagesMovedSoFar) => + logger.debug("Trying to move a single message to {} ({} messages left)", destinationQueue, messageQueue.size) + messageQueue.pop match { + case Some(internalMessage) => + messageMoveTaskState = MovingMessagesInProgress(numMessagesMovedSoFar + 1) + destinationQueue ! SendMessage(internalMessage.toNewMessageData) + maxNumberOfMessagesPerSecond match { + case Some(v) => + val nanosInSecond = 1.second.toNanos.toDouble + val delayNanos = (nanosInSecond / v).toLong + val delay = FiniteDuration(delayNanos, NANOSECONDS) + context.system.scheduler.scheduleOnce( + delay, + context.self, + MoveFirstMessage(taskId, destinationQueue, maxNumberOfMessagesPerSecond, queueManager) + ) + case None => + context.self ! MoveFirstMessage(taskId, destinationQueue, maxNumberOfMessagesPerSecond, queueManager) + } + ResultWithEvents.onlyEvents(List(QueueEvent.MessageRemoved(queueData.name, internalMessage.id))) case None => - context.self ! MoveFirstMessageToQueue(destinationQueue, maxNumberOfMessagesPerSecond) + logger.debug("No more messages to move") + messageMoveTaskState = NotMovingMessages + queueManager ! MessageMoveTaskFinished(taskId) + ResultWithEvents.empty } - ResultWithEvents.onlyEvents(List(QueueEvent.MessageRemoved(queueData.name, internalMessage.id))) - case None => - ResultWithEvents.empty + } + } + + def cancelMovingMessages(): Int = { + val numMessagesMoved = messageMoveTaskState match { + case NotMovingMessages => 0 + case MovingMessagesInProgress(numMessagesMoved) => numMessagesMoved } + messageMoveTaskState = NotMovingMessages + numMessagesMoved } } diff --git a/core/src/main/scala/org/elasticmq/msg/QueueManagerMsg.scala b/core/src/main/scala/org/elasticmq/msg/QueueManagerMsg.scala index c85002d96..0bbb9b761 100644 --- a/core/src/main/scala/org/elasticmq/msg/QueueManagerMsg.scala +++ b/core/src/main/scala/org/elasticmq/msg/QueueManagerMsg.scala @@ -15,4 +15,10 @@ case class StartMessageMoveTask( sourceQueue: ActorRef, destinationQueue: Option[ActorRef], maxNumberOfMessagesPerSecond: Option[Int] -) extends QueueManagerMsg[Either[ElasticMQError, StartMessageMoveTaskId]] +) extends QueueManagerMsg[Either[ElasticMQError, MessageMoveTaskId]] +case class MessageMoveTaskFinished( + taskHandle: MessageMoveTaskId +) extends QueueManagerMsg[Unit] +case class CancelMessageMoveTask( + taskHandle: MessageMoveTaskId +) extends QueueManagerMsg[Either[ElasticMQError, Int]] diff --git a/core/src/main/scala/org/elasticmq/msg/QueueMsg.scala b/core/src/main/scala/org/elasticmq/msg/QueueMsg.scala index fb48d5260..a686f53dc 100644 --- a/core/src/main/scala/org/elasticmq/msg/QueueMsg.scala +++ b/core/src/main/scala/org/elasticmq/msg/QueueMsg.scala @@ -54,8 +54,12 @@ case class DeleteMessage(deliveryReceipt: DeliveryReceipt) extends QueueMessageM case class LookupMessage(messageId: MessageId) extends QueueMessageMsg[Option[MessageData]] case object DeduplicationIdsCleanup extends QueueMessageMsg[Unit] case class RestoreMessages(messages: List[InternalMessage]) extends QueueMessageMsg[Unit] -case class StartMessageMoveTaskToQueue(destinationQueue: ActorRef, maxNumberOfMessagesPerSecond: Option[Int]) - extends QueueMessageMsg[StartMessageMoveTaskId] - -case class MoveFirstMessageToQueue(destinationQueue: ActorRef, maxNumberOfMessagesPerSecond: Option[Int]) - extends QueueMessageMsg[Unit] +case class StartMovingMessages(destinationQueue: ActorRef, maxNumberOfMessagesPerSecond: Option[Int], queueManager: ActorRef) + extends QueueMessageMsg[MessageMoveTaskId] +case class CancelMovingMessages() extends QueueMessageMsg[Int] +case class MoveFirstMessage( + taskId: MessageMoveTaskId, + destinationQueue: ActorRef, + maxNumberOfMessagesPerSecond: Option[Int], + queueManager: ActorRef +) extends QueueMessageMsg[Unit] diff --git a/core/src/main/scala/org/elasticmq/msg/package.scala b/core/src/main/scala/org/elasticmq/msg/package.scala index 894d7ec53..421fbffa4 100644 --- a/core/src/main/scala/org/elasticmq/msg/package.scala +++ b/core/src/main/scala/org/elasticmq/msg/package.scala @@ -1,5 +1,5 @@ package org.elasticmq package object msg { - type StartMessageMoveTaskId = String + type MessageMoveTaskId = String } diff --git a/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/Action.scala b/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/Action.scala index de793b676..e392df2be 100644 --- a/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/Action.scala +++ b/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/Action.scala @@ -22,4 +22,5 @@ object Action extends Enumeration { val TagQueue = Value("TagQueue") val UntagQueue = Value("UntagQueue") val StartMessageMoveTask = Value("StartMessageMoveTask") + val CancelMessageMoveTask = Value("CancelMessageMoveTask") } diff --git a/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/CancelMessageMoveTaskDirectives.scala b/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/CancelMessageMoveTaskDirectives.scala new file mode 100644 index 000000000..25838cd95 --- /dev/null +++ b/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/CancelMessageMoveTaskDirectives.scala @@ -0,0 +1,66 @@ +package org.elasticmq.rest.sqs + +import org.apache.pekko.http.scaladsl.server.Route +import org.elasticmq.ElasticMQError +import org.elasticmq.actor.reply._ +import org.elasticmq.msg.CancelMessageMoveTask +import org.elasticmq.rest.sqs.Action.{CancelMessageMoveTask => CancelMessageMoveTaskAction} +import org.elasticmq.rest.sqs.Constants._ +import org.elasticmq.rest.sqs.directives.ElasticMQDirectives +import org.elasticmq.rest.sqs.model.RequestPayload +import spray.json.DefaultJsonProtocol._ +import spray.json.RootJsonFormat + +import scala.async.Async._ + +trait CancelMessageMoveTaskDirectives { this: ElasticMQDirectives with QueueURLModule with ResponseMarshaller => + + def cancelMessageMoveTask(p: RequestPayload)(implicit marshallerDependencies: MarshallerDependencies): Route = { + p.action(CancelMessageMoveTaskAction) { + val params = p.as[CancelMessageMoveTaskRequest] + async { + await( + queueManagerActor ? CancelMessageMoveTask(params.TaskHandle) + ) match { + case Left(e: ElasticMQError) => throw new SQSException(e.code, errorMessage = Some(e.message)) + case Right(approximateNumberOfMessagesMoved) => + complete(CancelMessageMoveTaskResponse(approximateNumberOfMessagesMoved)) + } + } + } + } +} + +case class CancelMessageMoveTaskRequest( + TaskHandle: String +) + +object CancelMessageMoveTaskRequest { + implicit val requestJsonFormat: RootJsonFormat[CancelMessageMoveTaskRequest] = jsonFormat1( + CancelMessageMoveTaskRequest.apply + ) + + implicit val requestParamReader: FlatParamsReader[CancelMessageMoveTaskRequest] = + new FlatParamsReader[CancelMessageMoveTaskRequest] { + override def read(params: Map[String, String]): CancelMessageMoveTaskRequest = { + new CancelMessageMoveTaskRequest( + requiredParameter(params)(TaskHandleParameter) + ) + } + } +} + +case class CancelMessageMoveTaskResponse(ApproximateNumberOfMessagesMoved: Int) + +object CancelMessageMoveTaskResponse { + implicit val format: RootJsonFormat[CancelMessageMoveTaskResponse] = jsonFormat1(CancelMessageMoveTaskResponse.apply) + + implicit val xmlSerializer: XmlSerializer[CancelMessageMoveTaskResponse] = t => + + {t.ApproximateNumberOfMessagesMoved} + + + {EmptyRequestId} + + +} diff --git a/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/SQSRestServerBuilder.scala b/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/SQSRestServerBuilder.scala index b4377e366..a3b726ca9 100644 --- a/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/SQSRestServerBuilder.scala +++ b/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/SQSRestServerBuilder.scala @@ -1,24 +1,18 @@ package org.elasticmq.rest.sqs +import com.typesafe.config.ConfigFactory import org.apache.pekko.actor.{ActorRef, ActorSystem, Props} import org.apache.pekko.http.scaladsl.Http import org.apache.pekko.http.scaladsl.server.{Directive1, Directives} import org.apache.pekko.stream.ActorMaterializer import org.apache.pekko.util.Timeout -import com.typesafe.config.ConfigFactory import org.elasticmq._ import org.elasticmq.actor.QueueManagerActor import org.elasticmq.metrics.QueuesMetrics import org.elasticmq.rest.sqs.Constants._ import org.elasticmq.rest.sqs.XmlNsVersion.extractXmlNs -import org.elasticmq.rest.sqs.directives.{ - AWSProtocolDirectives, - AnyParamDirectives, - ElasticMQDirectives, - UnmatchedActionRoutes -} +import org.elasticmq.rest.sqs.directives.{AWSProtocolDirectives, AnyParamDirectives, ElasticMQDirectives, UnmatchedActionRoutes} import org.elasticmq.rest.sqs.model.RequestPayload -import org.elasticmq.rest.sqs.{AWSProtocol, XmlNsVersion} import org.elasticmq.util.{Logging, NowProvider} import java.io.ByteArrayOutputStream @@ -172,7 +166,8 @@ case class TheSQSRestServerBuilder( with ResponseMarshaller with QueueAttributesOps with ListDeadLetterSourceQueuesDirectives - with StartMessageMoveTaskDirectives { + with StartMessageMoveTaskDirectives + with CancelMessageMoveTaskDirectives { def serverAddress = currentServerAddress.get() @@ -214,6 +209,7 @@ case class TheSQSRestServerBuilder( listQueueTags(p) ~ listDeadLetterSourceQueues(p) ~ startMessageMoveTask(p) ~ + cancelMessageMoveTask(p) ~ // 4. Unmatched action unmatchedAction(p) @@ -345,6 +341,7 @@ object Constants { val SourceArnParameter = "SourceArn" val DestinationArnParameter = "DestinationArn" val MaxNumberOfMessagesPerSecondParameter = "MaxNumberOfMessagesPerSecond" + val TaskHandleParameter = "TaskHandle" } object ParametersUtil { diff --git a/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/StartMessageMoveTaskDirectives.scala b/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/StartMessageMoveTaskDirectives.scala index 206152dd7..4d96fc88a 100644 --- a/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/StartMessageMoveTaskDirectives.scala +++ b/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/StartMessageMoveTaskDirectives.scala @@ -1,19 +1,19 @@ package org.elasticmq.rest.sqs -import Constants._ import org.apache.pekko.actor.ActorRef -import org.apache.pekko.http.scaladsl.server.{Directive1, Route} +import org.apache.pekko.http.scaladsl.server.Route import org.elasticmq.ElasticMQError import org.elasticmq.actor.reply._ - -import scala.async.Async._ import org.elasticmq.msg.StartMessageMoveTask import org.elasticmq.rest.sqs.Action.{StartMessageMoveTask => StartMessageMoveTaskAction} +import org.elasticmq.rest.sqs.Constants._ import org.elasticmq.rest.sqs.directives.ElasticMQDirectives import org.elasticmq.rest.sqs.model.RequestPayload import spray.json.DefaultJsonProtocol._ import spray.json.RootJsonFormat +import scala.async.Async._ + trait StartMessageMoveTaskDirectives { this: ElasticMQDirectives with QueueURLModule with ResponseMarshaller => private val Arn = "(?:.+:(.+)?:(.+)?:)?(.+)".r @@ -51,9 +51,7 @@ trait StartMessageMoveTaskDirectives { this: ElasticMQDirectives with QueueURLMo queueManagerActor ? StartMessageMoveTask(sourceQueue, destinationQueue, maxNumberOfMessagesPerSecond) ) match { case Left(e: ElasticMQError) => throw new SQSException(e.code, errorMessage = Some(e.message)) - case Right(taskHandle) => - println("XXX" + taskHandle) - complete(StartMessageMoveTaskResponse(taskHandle)) + case Right(taskHandle) => complete(StartMessageMoveTaskResponse(taskHandle)) } } } From 508a567b6aa6048b1a314e9edc13b36153a28a02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Ossowski?= Date: Thu, 4 Apr 2024 10:43:41 +0200 Subject: [PATCH 04/12] fix missing ApproximateNumberOfMessagesMoved refactor test --- .../elasticmq/actor/QueueManagerActor.scala | 14 +- .../rest/sqs/MessageMoveTaskTest.scala | 143 +++++------------- 2 files changed, 51 insertions(+), 106 deletions(-) diff --git a/core/src/main/scala/org/elasticmq/actor/QueueManagerActor.scala b/core/src/main/scala/org/elasticmq/actor/QueueManagerActor.scala index d760c8783..0c6c7809c 100644 --- a/core/src/main/scala/org/elasticmq/actor/QueueManagerActor.scala +++ b/core/src/main/scala/org/elasticmq/actor/QueueManagerActor.scala @@ -111,9 +111,17 @@ class QueueManagerActor(nowProvider: NowProvider, limits: Limits, queueEventList logger.info("Cancelling message move task {}", taskHandle) messageMoveTasks.get(taskHandle) match { case Some(sourceQueue) => - sourceQueue ! CancelMovingMessages() - messageMoveTasks.remove(taskHandle) - ReplyWith(Right(0)) + val replyTo = context.sender() + sourceQueue ? CancelMovingMessages() onComplete { + case Success(numMessageMoved) => + logger.debug("Message move task {} cancelled", taskHandle) + messageMoveTasks.remove(taskHandle) + replyTo ! Right(numMessageMoved) + case Failure(ex) => + logger.error("Failed to cancel message move task", ex) + replyTo ! Left(ex) + } + DoNotReply() case None => ReplyWith(Left(new InvalidMessageMoveTaskId(taskHandle))) } diff --git a/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/MessageMoveTaskTest.scala b/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/MessageMoveTaskTest.scala index bd093eb00..32bac7763 100644 --- a/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/MessageMoveTaskTest.scala +++ b/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/MessageMoveTaskTest.scala @@ -1,115 +1,44 @@ package org.elasticmq.rest.sqs -import com.amazonaws.AmazonServiceException -import com.amazonaws.services.sqs.AmazonSQS import com.amazonaws.services.sqs.model.QueueAttributeName.ApproximateNumberOfMessages import com.amazonaws.services.sqs.model._ -import org.apache.http.HttpHost -import org.apache.http.client.entity.UrlEncodedFormEntity -import org.apache.http.client.methods.{HttpGet, HttpPost} -import org.apache.http.message.BasicNameValuePair -import org.apache.pekko.http.scaladsl.model.StatusCodes -import org.elasticmq._ import org.elasticmq.rest.sqs.model.RedrivePolicy import org.elasticmq.rest.sqs.model.RedrivePolicyJson.format import org.scalatest.concurrent.Eventually import org.scalatest.matchers.should.Matchers import spray.json.enrichAny -import java.net.URI -import java.nio.ByteBuffer -import java.util.UUID -import scala.collection.JavaConverters._ import scala.concurrent.duration.DurationInt -import scala.io.Source -import scala.util.control.Exception._ class MessageMoveTaskTest extends SqsClientServerCommunication with Matchers with Eventually { + private val NumMessages = 6 + private val DlqArn = s"arn:aws:sqs:$awsRegion:$awsAccountId:testQueue-dlq" + test("should run message move task") { // given - val dlq = client.createQueue(new CreateQueueRequest("testQueue-dlq")) - val redrivePolicy = RedrivePolicy("testQueue-dlq", awsRegion, awsAccountId, 1).toJson.toString() - val queue = - client.createQueue( - new CreateQueueRequest("testQueue") - .addAttributesEntry(redrivePolicyAttribute, redrivePolicy) - .addAttributesEntry("VisibilityTimeout", "1") - ) - - // when: send messages - val numMessages = 6 - for (i <- 0 until numMessages) { - client.sendMessage(queue.getQueueUrl, "Test message " + i) - } - - // and: receive messages - for (i <- 0 until numMessages) { - client.receiveMessage(queue.getQueueUrl) - } - - // and: receive messages again to make them move to DLQ - Thread.sleep(1500) - for (i <- 0 until numMessages) { - client.receiveMessage(queue.getQueueUrl) - } - - // then: ensure that messages are in DLQ - eventually(timeout(2.seconds), interval(100.millis)) { - fetchApproximateNumberOfMessages(queue.getQueueUrl) shouldEqual 0 - fetchApproximateNumberOfMessages(dlq.getQueueUrl) shouldEqual numMessages - } + val (queue, dlq) = createQueuesAndPopulateDlq // when: start message move task client.startMessageMoveTask( - new StartMessageMoveTaskRequest().withSourceArn(s"arn:aws:sqs:$awsRegion:$awsAccountId:testQueue-dlq") + new StartMessageMoveTaskRequest().withSourceArn(DlqArn) ) // then: ensure that messages are moved back to the original queue eventually(timeout(5.seconds), interval(100.millis)) { - fetchApproximateNumberOfMessages(queue.getQueueUrl) shouldEqual numMessages + fetchApproximateNumberOfMessages(queue.getQueueUrl) shouldEqual NumMessages fetchApproximateNumberOfMessages(dlq.getQueueUrl) shouldEqual 0 } } test("should run message move task with max number of messages per second") { // given - val dlq = client.createQueue(new CreateQueueRequest("testQueue-dlq")) - val redrivePolicy = RedrivePolicy("testQueue-dlq", awsRegion, awsAccountId, 1).toJson.toString() - val queue = - client.createQueue( - new CreateQueueRequest("testQueue") - .addAttributesEntry(redrivePolicyAttribute, redrivePolicy) - .addAttributesEntry("VisibilityTimeout", "1") - ) - - // when: send messages - val numMessages = 6 - for (i <- 0 until numMessages) { - client.sendMessage(queue.getQueueUrl, "Test message " + i) - } - - // and: receive messages - for (i <- 0 until numMessages) { - client.receiveMessage(queue.getQueueUrl) - } - - // and: receive messages again to make them move to DLQ - Thread.sleep(1500) - for (i <- 0 until numMessages) { - client.receiveMessage(queue.getQueueUrl) - } - - // then: ensure that messages are in DLQ - eventually(timeout(2.seconds), interval(100.millis)) { - fetchApproximateNumberOfMessages(queue.getQueueUrl) shouldEqual 0 - fetchApproximateNumberOfMessages(dlq.getQueueUrl) shouldEqual numMessages - } + val (queue, dlq) = createQueuesAndPopulateDlq // when: start message move task client.startMessageMoveTask( new StartMessageMoveTaskRequest() - .withSourceArn(s"arn:aws:sqs:$awsRegion:$awsAccountId:testQueue-dlq") + .withSourceArn(DlqArn) .withMaxNumberOfMessagesPerSecond(1) ) @@ -121,6 +50,33 @@ class MessageMoveTaskTest extends SqsClientServerCommunication with Matchers wit test("should run and cancel message move task") { // given + val (queue, dlq) = createQueuesAndPopulateDlq + + // when: start message move task + val taskHandle = client.startMessageMoveTask( + new StartMessageMoveTaskRequest() + .withSourceArn(DlqArn) + .withMaxNumberOfMessagesPerSecond(1) + ).getTaskHandle + + // and: cancel the task after 2 seconds + Thread.sleep(2000) + val numMessagesMoved = client.cancelMessageMoveTask(new CancelMessageMoveTaskRequest().withTaskHandle(taskHandle)) + .getApproximateNumberOfMessagesMoved + + // and: fetch ApproximateNumberOfMessages + val numMessagesInMainQueue = fetchApproximateNumberOfMessages(queue.getQueueUrl) + val numMessagesInDlQueue = fetchApproximateNumberOfMessages(dlq.getQueueUrl) + + numMessagesMoved shouldEqual numMessagesInMainQueue + + // then: ApproximateNumberOfMessages should not change after 2 seconds + Thread.sleep(2000) + fetchApproximateNumberOfMessages(queue.getQueueUrl) shouldEqual numMessagesInMainQueue + fetchApproximateNumberOfMessages(dlq.getQueueUrl) shouldEqual numMessagesInDlQueue + } + + private def createQueuesAndPopulateDlq: (CreateQueueResult, CreateQueueResult) = { val dlq = client.createQueue(new CreateQueueRequest("testQueue-dlq")) val redrivePolicy = RedrivePolicy("testQueue-dlq", awsRegion, awsAccountId, 1).toJson.toString() val queue = @@ -131,47 +87,28 @@ class MessageMoveTaskTest extends SqsClientServerCommunication with Matchers wit ) // when: send messages - val numMessages = 6 - for (i <- 0 until numMessages) { + for (i <- 0 until NumMessages) { client.sendMessage(queue.getQueueUrl, "Test message " + i) } // and: receive messages - for (i <- 0 until numMessages) { + for (i <- 0 until NumMessages) { client.receiveMessage(queue.getQueueUrl) } // and: receive messages again to make them move to DLQ Thread.sleep(1500) - for (i <- 0 until numMessages) { + for (i <- 0 until NumMessages) { client.receiveMessage(queue.getQueueUrl) } // then: ensure that messages are in DLQ eventually(timeout(2.seconds), interval(100.millis)) { fetchApproximateNumberOfMessages(queue.getQueueUrl) shouldEqual 0 - fetchApproximateNumberOfMessages(dlq.getQueueUrl) shouldEqual numMessages + fetchApproximateNumberOfMessages(dlq.getQueueUrl) shouldEqual NumMessages } - // when: start message move task - val taskHandle = client.startMessageMoveTask( - new StartMessageMoveTaskRequest() - .withSourceArn(s"arn:aws:sqs:$awsRegion:$awsAccountId:testQueue-dlq") - .withMaxNumberOfMessagesPerSecond(1) - ).getTaskHandle - - // and: cancel the task after 2 seconds - Thread.sleep(2000) - client.cancelMessageMoveTask(new CancelMessageMoveTaskRequest().withTaskHandle(taskHandle)) - - // and: fetch ApproximateNumberOfMessages - val numMessagesInMainQueue = fetchApproximateNumberOfMessages(queue.getQueueUrl) - val numMessagesInDlQueue = fetchApproximateNumberOfMessages(dlq.getQueueUrl) - - // then: ApproximateNumberOfMessages should not change after 2 seconds - Thread.sleep(2000) - fetchApproximateNumberOfMessages(queue.getQueueUrl) shouldEqual numMessagesInMainQueue - fetchApproximateNumberOfMessages(dlq.getQueueUrl) shouldEqual numMessagesInDlQueue + (queue, dlq) } private def fetchApproximateNumberOfMessages(queueUrl: String): Int = { From 6a06804c8821c198d4d20bf6e5e509eb095272eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Ossowski?= Date: Fri, 5 Apr 2024 09:20:41 +0200 Subject: [PATCH 05/12] ListMessageMoveTasksDirectives and error handling --- .../scala/org/elasticmq/ElasticMQError.scala | 9 +- .../elasticmq/actor/QueueManagerActor.scala | 34 +++- .../actor/queue/QueueActorMessageOps.scala | 16 +- .../operations/MoveMessagesAsyncOps.scala | 118 +++++++++++-- .../org/elasticmq/msg/QueueManagerMsg.scala | 4 +- .../scala/org/elasticmq/msg/QueueMsg.scala | 15 +- .../rest/sqs/MessageMoveTaskTest.scala | 163 ++++++++++++++++-- .../scala/org/elasticmq/rest/sqs/Action.scala | 1 + .../org/elasticmq/rest/sqs/ArnSupport.scala | 12 ++ .../sqs/CancelMessageMoveTaskDirectives.scala | 2 +- .../sqs/ListMessageMoveTasksDirectives.scala | 114 ++++++++++++ .../org/elasticmq/rest/sqs/SQSException.scala | 11 +- .../rest/sqs/SQSRestServerBuilder.scala | 4 +- .../sqs/StartMessageMoveTaskDirectives.scala | 40 ++--- 14 files changed, 461 insertions(+), 82 deletions(-) create mode 100644 rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/ArnSupport.scala create mode 100644 rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/ListMessageMoveTasksDirectives.scala diff --git a/core/src/main/scala/org/elasticmq/ElasticMQError.scala b/core/src/main/scala/org/elasticmq/ElasticMQError.scala index 7a70bc442..1cef1ac94 100644 --- a/core/src/main/scala/org/elasticmq/ElasticMQError.scala +++ b/core/src/main/scala/org/elasticmq/ElasticMQError.scala @@ -3,7 +3,7 @@ import org.elasticmq.msg.MessageMoveTaskId trait ElasticMQError { val queueName: String - val code: String + val code: String // TODO: code should be handled in rest-sqs module val message: String } @@ -33,8 +33,13 @@ class InvalidReceiptHandle(val queueName: String, receiptHandle: String) extends } class InvalidMessageMoveTaskId(val taskId: MessageMoveTaskId) extends ElasticMQError { - val code = "InvalidMessageMoveTaskId" + val code = "ResourceNotFoundException" val message = s"""The task id "$taskId" is not valid or does not exist""" override val queueName: String = "invalid" } + +class MessageMoveTaskAlreadyRunning(val queueName: String) extends ElasticMQError { + val code = "AWS.SimpleQueueService.UnsupportedOperation" + val message = s"""A message move task is already running on queue "$queueName"""" +} diff --git a/core/src/main/scala/org/elasticmq/actor/QueueManagerActor.scala b/core/src/main/scala/org/elasticmq/actor/QueueManagerActor.scala index 0c6c7809c..a2c8515a4 100644 --- a/core/src/main/scala/org/elasticmq/actor/QueueManagerActor.scala +++ b/core/src/main/scala/org/elasticmq/actor/QueueManagerActor.scala @@ -73,7 +73,13 @@ class QueueManagerActor(nowProvider: NowProvider, limits: Limits, queueEventList case (name, actor) if actor.queueData.deadLettersQueue.exists(_.name == queueName) => name }.toList - case StartMessageMoveTask(sourceQueue, destinationQueue, maxNumberOfMessagesPerSecond) => + case StartMessageMoveTask( + sourceQueue, + sourceArn, + destinationQueue, + destinationArn, + maxNumberOfMessagesPerSecond + ) => val replyTo = sender() val destination = destinationQueue.map(Future.successful).getOrElse { val queueDataF = sourceQueue ? GetQueueData() @@ -89,15 +95,27 @@ class QueueManagerActor(nowProvider: NowProvider, limits: Limits, queueEventList } destination .flatMap(destinationQueueActorRef => { - val taskIdF = - sourceQueue ? StartMovingMessages(destinationQueueActorRef, maxNumberOfMessagesPerSecond, self) - taskIdF.map(taskId => (taskId, destinationQueueActorRef)) + val resultF = + sourceQueue ? StartMovingMessages( + destinationQueueActorRef, + destinationArn, + sourceArn, + maxNumberOfMessagesPerSecond, + self + ) + resultF.map(result => (result, destinationQueueActorRef)) }) .onComplete { - case Success((taskId, destinationQueueActorRef)) => - logger.debug("Message move task {} => {} created", sourceQueue, destinationQueueActorRef) - messageMoveTasks.put(taskId, sourceQueue) - replyTo ! Right(taskId) + case Success((result, destinationQueueActorRef)) => + result match { + case Right(taskId) => + logger.debug("Message move task {} => {} created", sourceQueue, destinationQueueActorRef) + messageMoveTasks.put(taskId, sourceQueue) + replyTo ! Right(taskId) + case Left(error) => + logger.error("Failed to start message move task: {}", error) + replyTo ! Left(error) + } case Failure(ex) => logger.error("Failed to start message move task", ex) } DoNotReply() diff --git a/core/src/main/scala/org/elasticmq/actor/queue/QueueActorMessageOps.scala b/core/src/main/scala/org/elasticmq/actor/queue/QueueActorMessageOps.scala index 17bd05b44..1a8920798 100644 --- a/core/src/main/scala/org/elasticmq/actor/queue/QueueActorMessageOps.scala +++ b/core/src/main/scala/org/elasticmq/actor/queue/QueueActorMessageOps.scala @@ -39,12 +39,20 @@ trait QueueActorMessageOps fifoMessagesHistory = fifoMessagesHistory.cleanOutdatedMessages(nowProvider) DoNotReply() case RestoreMessages(messages) => restoreMessages(messages) - case StartMovingMessages(destinationQueue, maxNumberOfMessagesPerSecond, queueManager) => - startMovingMessages(destinationQueue, maxNumberOfMessagesPerSecond, queueManager) + case StartMovingMessages( + destinationQueue, + destinationArn, + sourceArn, + maxNumberOfMessagesPerSecond, + queueManager + ) => + startMovingMessages(destinationQueue, destinationArn, sourceArn, maxNumberOfMessagesPerSecond, queueManager) case CancelMovingMessages() => cancelMovingMessages() - case MoveFirstMessage(taskId, destinationQueue, maxNumberOfMessagesPerSecond, queueManager) => - moveFirstMessage(taskId, destinationQueue, maxNumberOfMessagesPerSecond, queueManager).send() + case MoveFirstMessage(destinationQueue, queueManager) => + moveFirstMessage(destinationQueue, queueManager).send() + case GetMovingMessagesTasks() => + getMovingMessagesTasks } } } diff --git a/core/src/main/scala/org/elasticmq/actor/queue/operations/MoveMessagesAsyncOps.scala b/core/src/main/scala/org/elasticmq/actor/queue/operations/MoveMessagesAsyncOps.scala index a4fb8f3ba..73c946b55 100644 --- a/core/src/main/scala/org/elasticmq/actor/queue/operations/MoveMessagesAsyncOps.scala +++ b/core/src/main/scala/org/elasticmq/actor/queue/operations/MoveMessagesAsyncOps.scala @@ -4,46 +4,88 @@ import org.apache.pekko.actor.ActorRef import org.elasticmq.actor.queue.{QueueActorStorage, QueueEvent} import org.elasticmq.msg.{MessageMoveTaskFinished, MessageMoveTaskId, MoveFirstMessage, SendMessage} import org.elasticmq.util.Logging +import org.elasticmq.{ElasticMQError, MessageMoveTaskAlreadyRunning} import java.util.UUID import scala.concurrent.duration.{DurationInt, FiniteDuration, NANOSECONDS} sealed trait MessageMoveTaskState case object NotMovingMessages extends MessageMoveTaskState -case class MovingMessagesInProgress(numMessagesMoved: Int) extends MessageMoveTaskState +case class MovingMessagesInProgress( + numberOfMessagesMoved: Long, + numberOfMessagesToMove: Long, + destinationArn: Option[String], + maxNumberOfMessagesPerSecond: Option[Int], + sourceArn: String, + startedTimestamp: Long, + taskHandle: MessageMoveTaskId +) extends MessageMoveTaskState + +case class MessageMoveTaskData( + numberOfMessagesMoved: Long, + numberOfMessagesToMove: Long, + destinationArn: Option[String], + maxNumberOfMessagesPerSecond: Option[Int], + sourceArn: String, + startedTimestamp: Long, + status: String, // RUNNING, COMPLETED, CANCELLING, CANCELLED, and FAILED + taskHandle: MessageMoveTaskId +) trait MoveMessagesAsyncOps extends Logging { this: QueueActorStorage => + private val prevMessageMoveTasks = collection.mutable.Buffer[MessageMoveTaskData]() private var messageMoveTaskState: MessageMoveTaskState = NotMovingMessages def startMovingMessages( destinationQueue: ActorRef, + destinationArn: Option[String], + sourceArn: String, maxNumberOfMessagesPerSecond: Option[Int], queueManager: ActorRef - ): MessageMoveTaskId = { - val taskId = UUID.randomUUID().toString - logger.debug("Starting message move task to queue {} (task id: {})", destinationQueue, taskId) - messageMoveTaskState = MovingMessagesInProgress(0) - context.self ! MoveFirstMessage(taskId, destinationQueue, maxNumberOfMessagesPerSecond, queueManager) - taskId + ): Either[ElasticMQError, MessageMoveTaskId] = { + messageMoveTaskState match { + case NotMovingMessages => + val taskHandle = UUID.randomUUID().toString + logger.debug("Starting message move task to queue {} (task handle: {})", destinationQueue, taskHandle) + messageMoveTaskState = MovingMessagesInProgress( + 0, + messageQueue.size, + destinationArn, + maxNumberOfMessagesPerSecond, + sourceArn, + startedTimestamp = System.currentTimeMillis(), + taskHandle + ) + context.self ! MoveFirstMessage(destinationQueue, queueManager) + Right(taskHandle) + case _: MovingMessagesInProgress => + Left(new MessageMoveTaskAlreadyRunning(queueData.name)) + } } def moveFirstMessage( - taskId: MessageMoveTaskId, destinationQueue: ActorRef, - maxNumberOfMessagesPerSecond: Option[Int], queueManager: ActorRef ): ResultWithEvents[Unit] = { messageMoveTaskState match { case NotMovingMessages => - logger.debug("Moving messages task {} was finished or cancelled", taskId) + logger.debug("Not moving messages") ResultWithEvents.empty - case MovingMessagesInProgress(numMessagesMovedSoFar) => + case mmInProgress @ MovingMessagesInProgress( + numberOfMessagesMoved, + _, + destinationArn, + maxNumberOfMessagesPerSecond, + sourceArn, + startedTimestamp, + taskHandle + ) => logger.debug("Trying to move a single message to {} ({} messages left)", destinationQueue, messageQueue.size) messageQueue.pop match { case Some(internalMessage) => - messageMoveTaskState = MovingMessagesInProgress(numMessagesMovedSoFar + 1) + messageMoveTaskState = mmInProgress.copy(numberOfMessagesMoved = numberOfMessagesMoved + 1) destinationQueue ! SendMessage(internalMessage.toNewMessageData) maxNumberOfMessagesPerSecond match { case Some(v) => @@ -53,27 +95,65 @@ trait MoveMessagesAsyncOps extends Logging { context.system.scheduler.scheduleOnce( delay, context.self, - MoveFirstMessage(taskId, destinationQueue, maxNumberOfMessagesPerSecond, queueManager) + MoveFirstMessage(destinationQueue, queueManager) ) case None => - context.self ! MoveFirstMessage(taskId, destinationQueue, maxNumberOfMessagesPerSecond, queueManager) + context.self ! MoveFirstMessage(destinationQueue, queueManager) } ResultWithEvents.onlyEvents(List(QueueEvent.MessageRemoved(queueData.name, internalMessage.id))) case None => logger.debug("No more messages to move") + prevMessageMoveTasks += MessageMoveTaskData( + numberOfMessagesMoved, + numberOfMessagesToMove = 0, + destinationArn, + maxNumberOfMessagesPerSecond, + sourceArn, + startedTimestamp, + status = "COMPLETED", + taskHandle + ) messageMoveTaskState = NotMovingMessages - queueManager ! MessageMoveTaskFinished(taskId) + queueManager ! MessageMoveTaskFinished(taskHandle) ResultWithEvents.empty } - } } + } - def cancelMovingMessages(): Int = { + def cancelMovingMessages(): Long = { val numMessagesMoved = messageMoveTaskState match { - case NotMovingMessages => 0 - case MovingMessagesInProgress(numMessagesMoved) => numMessagesMoved + case NotMovingMessages => 0 + case mmInProgress: MovingMessagesInProgress => mmInProgress.numberOfMessagesMoved } messageMoveTaskState = NotMovingMessages numMessagesMoved } + + def getMovingMessagesTasks: List[MessageMoveTaskData] = { + val runningTaskAsList = messageMoveTaskState match { + case NotMovingMessages => List.empty + case MovingMessagesInProgress( + numberOfMessagesMoved, + numberOfMessagesToMove, + destinationArn, + maxNumberOfMessagesPerSecond, + sourceArn, + startedTimestamp, + taskHandle + ) => + List( + MessageMoveTaskData( + numberOfMessagesMoved, + numberOfMessagesToMove, + destinationArn, + maxNumberOfMessagesPerSecond, + sourceArn, + startedTimestamp, + status = "RUNNING", + taskHandle + ) + ) + } + (prevMessageMoveTasks.toList ++ runningTaskAsList).reverse + } } diff --git a/core/src/main/scala/org/elasticmq/msg/QueueManagerMsg.scala b/core/src/main/scala/org/elasticmq/msg/QueueManagerMsg.scala index 0bbb9b761..1bb7501f4 100644 --- a/core/src/main/scala/org/elasticmq/msg/QueueManagerMsg.scala +++ b/core/src/main/scala/org/elasticmq/msg/QueueManagerMsg.scala @@ -13,7 +13,9 @@ case class ListQueues() extends QueueManagerMsg[Seq[String]] case class ListDeadLetterSourceQueues(queueName: String) extends QueueManagerMsg[List[String]] case class StartMessageMoveTask( sourceQueue: ActorRef, + sourceArn: String, destinationQueue: Option[ActorRef], + destinationArn: Option[String], maxNumberOfMessagesPerSecond: Option[Int] ) extends QueueManagerMsg[Either[ElasticMQError, MessageMoveTaskId]] case class MessageMoveTaskFinished( @@ -21,4 +23,4 @@ case class MessageMoveTaskFinished( ) extends QueueManagerMsg[Unit] case class CancelMessageMoveTask( taskHandle: MessageMoveTaskId -) extends QueueManagerMsg[Either[ElasticMQError, Int]] +) extends QueueManagerMsg[Either[ElasticMQError, Long]] diff --git a/core/src/main/scala/org/elasticmq/msg/QueueMsg.scala b/core/src/main/scala/org/elasticmq/msg/QueueMsg.scala index a686f53dc..faf3d8a54 100644 --- a/core/src/main/scala/org/elasticmq/msg/QueueMsg.scala +++ b/core/src/main/scala/org/elasticmq/msg/QueueMsg.scala @@ -3,6 +3,7 @@ package org.elasticmq.msg import org.apache.pekko.actor.ActorRef import org.elasticmq._ import org.elasticmq.actor.queue.InternalMessage +import org.elasticmq.actor.queue.operations.MessageMoveTaskData import org.elasticmq.actor.reply.Replyable import java.time.Duration @@ -54,12 +55,16 @@ case class DeleteMessage(deliveryReceipt: DeliveryReceipt) extends QueueMessageM case class LookupMessage(messageId: MessageId) extends QueueMessageMsg[Option[MessageData]] case object DeduplicationIdsCleanup extends QueueMessageMsg[Unit] case class RestoreMessages(messages: List[InternalMessage]) extends QueueMessageMsg[Unit] -case class StartMovingMessages(destinationQueue: ActorRef, maxNumberOfMessagesPerSecond: Option[Int], queueManager: ActorRef) - extends QueueMessageMsg[MessageMoveTaskId] -case class CancelMovingMessages() extends QueueMessageMsg[Int] -case class MoveFirstMessage( - taskId: MessageMoveTaskId, +case class StartMovingMessages( destinationQueue: ActorRef, + destinationArn: Option[String], + sourceArn: String, maxNumberOfMessagesPerSecond: Option[Int], queueManager: ActorRef +) extends QueueMessageMsg[Either[ElasticMQError, MessageMoveTaskId]] +case class CancelMovingMessages() extends QueueMessageMsg[Long] +case class MoveFirstMessage( + destinationQueue: ActorRef, + queueManager: ActorRef ) extends QueueMessageMsg[Unit] +case class GetMovingMessagesTasks() extends QueueMessageMsg[List[MessageMoveTaskData]] diff --git a/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/MessageMoveTaskTest.scala b/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/MessageMoveTaskTest.scala index 32bac7763..1f792d7fe 100644 --- a/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/MessageMoveTaskTest.scala +++ b/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/MessageMoveTaskTest.scala @@ -8,6 +8,7 @@ import org.scalatest.concurrent.Eventually import org.scalatest.matchers.should.Matchers import spray.json.enrichAny +import java.util.UUID import scala.concurrent.duration.DurationInt class MessageMoveTaskTest extends SqsClientServerCommunication with Matchers with Eventually { @@ -17,7 +18,7 @@ class MessageMoveTaskTest extends SqsClientServerCommunication with Matchers wit test("should run message move task") { // given - val (queue, dlq) = createQueuesAndPopulateDlq + val (queue, dlq) = createQueuesAndPopulateDlq() // when: start message move task client.startMessageMoveTask( @@ -33,7 +34,7 @@ class MessageMoveTaskTest extends SqsClientServerCommunication with Matchers wit test("should run message move task with max number of messages per second") { // given - val (queue, dlq) = createQueuesAndPopulateDlq + val (queue, dlq) = createQueuesAndPopulateDlq() // when: start message move task client.startMessageMoveTask( @@ -48,26 +49,138 @@ class MessageMoveTaskTest extends SqsClientServerCommunication with Matchers wit fetchApproximateNumberOfMessages(dlq.getQueueUrl) should (be > 1 and be < 6) } - test("should run and cancel message move task") { + test("should not run two message move tasks in parallel") { // given - val (queue, dlq) = createQueuesAndPopulateDlq + val (queue, dlq) = createQueuesAndPopulateDlq() // when: start message move task - val taskHandle = client.startMessageMoveTask( + client.startMessageMoveTask( new StartMessageMoveTaskRequest() .withSourceArn(DlqArn) .withMaxNumberOfMessagesPerSecond(1) - ).getTaskHandle + ) + + // and: try to start another message move task + val thrown = intercept[UnsupportedOperationException] { + client.startMessageMoveTask( + new StartMessageMoveTaskRequest() + .withSourceArn(DlqArn) + .withMaxNumberOfMessagesPerSecond(1) + ) + } + + // then + thrown.getErrorMessage shouldBe "A message move task is already running on queue \"testQueue-dlq\"" + + // and: ensure that not all messages were moved back to the original queue after 2 seconds + Thread.sleep(2000) + fetchApproximateNumberOfMessages(queue.getQueueUrl) should (be > 1 and be < 6) + fetchApproximateNumberOfMessages(dlq.getQueueUrl) should (be > 1 and be < 6) + } + + test("should run message move task and list it") { + // given + val (queue, dlq) = createQueuesAndPopulateDlq() + + // when + val taskHandle = client + .startMessageMoveTask( + new StartMessageMoveTaskRequest() + .withSourceArn(DlqArn) + .withMaxNumberOfMessagesPerSecond(1) + ) + .getTaskHandle + + // and + val results = + client.listMessageMoveTasks(new ListMessageMoveTasksRequest().withSourceArn(DlqArn).withMaxResults(10)).getResults + + // then + results.size shouldEqual 1 + results.get(0).getTaskHandle shouldBe taskHandle + results.get(0).getStatus shouldBe "RUNNING" + } + + test("should run two message move task and list them in the correct order") { + // given + val (queue, dlq) = createQueuesAndPopulateDlq() + + // when + val firstTaskHandle = client + .startMessageMoveTask( + new StartMessageMoveTaskRequest() + .withSourceArn(DlqArn) + ) + .getTaskHandle + + // and + receiveAllMessagesTwice(queue) + + // and + val secondTaskHandle = client + .startMessageMoveTask( + new StartMessageMoveTaskRequest() + .withSourceArn(DlqArn) + .withMaxNumberOfMessagesPerSecond(1) + ) + .getTaskHandle + + // and + val results = + client.listMessageMoveTasks(new ListMessageMoveTasksRequest().withSourceArn(DlqArn).withMaxResults(10)).getResults + + // then + results.size shouldEqual 2 + results.get(0).getTaskHandle shouldBe secondTaskHandle + results.get(0).getStatus shouldBe "RUNNING" + results.get(1).getTaskHandle shouldBe firstTaskHandle + results.get(1).getStatus shouldBe "COMPLETED" + } + + test("should fail to list tasks for non-existing source ARN") { + intercept[QueueDoesNotExistException] { + client.listMessageMoveTasks( + new ListMessageMoveTasksRequest() + .withSourceArn(s"arn:aws:sqs:$awsRegion:$awsAccountId:nonExistingQueue") + .withMaxResults(10) + ) + } + } + + test("should fail to list tasks for invalid ARN") { + intercept[QueueDoesNotExistException] { + client.listMessageMoveTasks( + new ListMessageMoveTasksRequest() + .withSourceArn("invalidArn") + .withMaxResults(10) + ) + } + } + + test("should run and cancel message move task") { + // given + val (queue, dlq) = createQueuesAndPopulateDlq() + + // when: start message move task + val taskHandle = client + .startMessageMoveTask( + new StartMessageMoveTaskRequest() + .withSourceArn(DlqArn) + .withMaxNumberOfMessagesPerSecond(1) + ) + .getTaskHandle // and: cancel the task after 2 seconds Thread.sleep(2000) - val numMessagesMoved = client.cancelMessageMoveTask(new CancelMessageMoveTaskRequest().withTaskHandle(taskHandle)) + val numMessagesMoved = client + .cancelMessageMoveTask(new CancelMessageMoveTaskRequest().withTaskHandle(taskHandle)) .getApproximateNumberOfMessagesMoved // and: fetch ApproximateNumberOfMessages val numMessagesInMainQueue = fetchApproximateNumberOfMessages(queue.getQueueUrl) val numMessagesInDlQueue = fetchApproximateNumberOfMessages(dlq.getQueueUrl) + // then numMessagesMoved shouldEqual numMessagesInMainQueue // then: ApproximateNumberOfMessages should not change after 2 seconds @@ -76,7 +189,15 @@ class MessageMoveTaskTest extends SqsClientServerCommunication with Matchers wit fetchApproximateNumberOfMessages(dlq.getQueueUrl) shouldEqual numMessagesInDlQueue } - private def createQueuesAndPopulateDlq: (CreateQueueResult, CreateQueueResult) = { + test("should fail to cancel non-existing task") { + intercept[ResourceNotFoundException] { + client + .cancelMessageMoveTask(new CancelMessageMoveTaskRequest().withTaskHandle(UUID.randomUUID().toString)) + .getApproximateNumberOfMessagesMoved + } + } + + private def createQueuesAndPopulateDlq(): (CreateQueueResult, CreateQueueResult) = { val dlq = client.createQueue(new CreateQueueRequest("testQueue-dlq")) val redrivePolicy = RedrivePolicy("testQueue-dlq", awsRegion, awsAccountId, 1).toJson.toString() val queue = @@ -91,16 +212,8 @@ class MessageMoveTaskTest extends SqsClientServerCommunication with Matchers wit client.sendMessage(queue.getQueueUrl, "Test message " + i) } - // and: receive messages - for (i <- 0 until NumMessages) { - client.receiveMessage(queue.getQueueUrl) - } - - // and: receive messages again to make them move to DLQ - Thread.sleep(1500) - for (i <- 0 until NumMessages) { - client.receiveMessage(queue.getQueueUrl) - } + // and: receive messages twice to make them move to DLQ + receiveAllMessagesTwice(queue) // then: ensure that messages are in DLQ eventually(timeout(2.seconds), interval(100.millis)) { @@ -111,12 +224,24 @@ class MessageMoveTaskTest extends SqsClientServerCommunication with Matchers wit (queue, dlq) } + private def receiveAllMessagesTwice(queue: CreateQueueResult): Unit = { + for (i <- 0 until NumMessages) { + client.receiveMessage(queue.getQueueUrl) + } + + Thread.sleep(1500) + for (i <- 0 until NumMessages) { + client.receiveMessage(queue.getQueueUrl) + } + } + private def fetchApproximateNumberOfMessages(queueUrl: String): Int = { client .getQueueAttributes( new GetQueueAttributesRequest().withQueueUrl(queueUrl).withAttributeNames(ApproximateNumberOfMessages) ) .getAttributes - .get(ApproximateNumberOfMessages.toString).toInt + .get(ApproximateNumberOfMessages.toString) + .toInt } } diff --git a/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/Action.scala b/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/Action.scala index e392df2be..5623b637a 100644 --- a/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/Action.scala +++ b/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/Action.scala @@ -23,4 +23,5 @@ object Action extends Enumeration { val UntagQueue = Value("UntagQueue") val StartMessageMoveTask = Value("StartMessageMoveTask") val CancelMessageMoveTask = Value("CancelMessageMoveTask") + val ListMessageMoveTasks = Value("ListMessageMoveTasks") } diff --git a/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/ArnSupport.scala b/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/ArnSupport.scala new file mode 100644 index 000000000..52fa8042b --- /dev/null +++ b/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/ArnSupport.scala @@ -0,0 +1,12 @@ +package org.elasticmq.rest.sqs + +trait ArnSupport { + + private val ArnPattern = "(?:.+:(.+)?:(.+)?:)?(.+)".r + + def extractQueueName(arn: String): String = + arn match { + case ArnPattern(_, _, queueName) => queueName + case _ => throw new SQSException("InvalidParameterValue") + } +} diff --git a/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/CancelMessageMoveTaskDirectives.scala b/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/CancelMessageMoveTaskDirectives.scala index 25838cd95..8aa78a2a8 100644 --- a/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/CancelMessageMoveTaskDirectives.scala +++ b/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/CancelMessageMoveTaskDirectives.scala @@ -50,7 +50,7 @@ object CancelMessageMoveTaskRequest { } } -case class CancelMessageMoveTaskResponse(ApproximateNumberOfMessagesMoved: Int) +case class CancelMessageMoveTaskResponse(ApproximateNumberOfMessagesMoved: Long) object CancelMessageMoveTaskResponse { implicit val format: RootJsonFormat[CancelMessageMoveTaskResponse] = jsonFormat1(CancelMessageMoveTaskResponse.apply) diff --git a/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/ListMessageMoveTasksDirectives.scala b/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/ListMessageMoveTasksDirectives.scala new file mode 100644 index 000000000..045fd8f48 --- /dev/null +++ b/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/ListMessageMoveTasksDirectives.scala @@ -0,0 +1,114 @@ +package org.elasticmq.rest.sqs + +import org.apache.pekko.http.scaladsl.server.Route +import org.elasticmq.actor.reply._ +import org.elasticmq.msg.GetMovingMessagesTasks +import org.elasticmq.rest.sqs.Action.{ListMessageMoveTasks => ListMessageMoveTasksAction} +import org.elasticmq.rest.sqs.Constants._ +import org.elasticmq.rest.sqs.directives.ElasticMQDirectives +import org.elasticmq.rest.sqs.model.RequestPayload +import spray.json.DefaultJsonProtocol._ +import spray.json.RootJsonFormat + +trait ListMessageMoveTasksDirectives extends ArnSupport { + this: ElasticMQDirectives with QueueURLModule with ResponseMarshaller => + + def listMessageMoveTasks(p: RequestPayload)(implicit marshallerDependencies: MarshallerDependencies): Route = { + p.action(ListMessageMoveTasksAction) { + val params = p.as[ListMessageMoveTasksRequest] + val sourceQueueName = extractQueueName(params.SourceArn) + queueActorAndDataFromQueueName(sourceQueueName) { (sourceQueue, _) => + for { + tasks <- sourceQueue ? GetMovingMessagesTasks() + } yield { + complete { + ListMessageMoveTasksResponse( + tasks.map(task => + MessageMoveTaskResponse( + task.numberOfMessagesMoved, + task.numberOfMessagesToMove, + task.destinationArn, + None, + task.maxNumberOfMessagesPerSecond, + task.sourceArn, + task.startedTimestamp, + task.status, + task.taskHandle + ) + ) + ) + } + } + } + } + } +} + +case class ListMessageMoveTasksRequest( + MaxResults: Option[Int], + SourceArn: String +) + +object ListMessageMoveTasksRequest { + implicit val requestJsonFormat: RootJsonFormat[ListMessageMoveTasksRequest] = jsonFormat2( + ListMessageMoveTasksRequest.apply + ) + + implicit val requestParamReader: FlatParamsReader[ListMessageMoveTasksRequest] = + new FlatParamsReader[ListMessageMoveTasksRequest] { + override def read(params: Map[String, String]): ListMessageMoveTasksRequest = { + new ListMessageMoveTasksRequest( + optionalParameter(params)(MaxResultsParameter).map(_.toInt), + requiredParameter(params)(SourceArnParameter) + ) + } + } +} + +case class MessageMoveTaskResponse( + ApproximateNumberOfMessagesMoved: Long, + ApproximateNumberOfMessagesToMove: Long, + DestinationArn: Option[String], + FailureReason: Option[String], + MaxNumberOfMessagesPerSecond: Option[Int], + SourceArn: String, + StartedTimestamp: Long, + Status: String, + TaskHandle: String +) +case class ListMessageMoveTasksResponse(Results: List[MessageMoveTaskResponse]) + +object ListMessageMoveTasksResponse { + implicit val format: RootJsonFormat[MessageMoveTaskResponse] = jsonFormat9(MessageMoveTaskResponse.apply) + implicit val formatList: RootJsonFormat[ListMessageMoveTasksResponse] = jsonFormat1( + ListMessageMoveTasksResponse.apply + ) + + implicit val xmlSerializer: XmlSerializer[ListMessageMoveTasksResponse] = t => + + { + t.Results.map(task => + + {task.ApproximateNumberOfMessagesMoved} + { + task.ApproximateNumberOfMessagesToMove + } + {task.DestinationArn.map(arn => {arn}).getOrElse("")} + { + task.MaxNumberOfMessagesPerSecond + .map(v => {v}) + .getOrElse("") + } + {task.SourceArn} + {task.StartedTimestamp} + {task.Status} + {task.TaskHandle} + + ) + } + + + {EmptyRequestId} + + +} diff --git a/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/SQSException.scala b/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/SQSException.scala index a9161e3b9..19edcfb89 100644 --- a/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/SQSException.scala +++ b/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/SQSException.scala @@ -1,8 +1,8 @@ package org.elasticmq.rest.sqs -import scala.xml.Elem +import org.elasticmq.rest.sqs.Constants._ -import Constants._ +import scala.xml.Elem class SQSException( val code: String, @@ -76,4 +76,11 @@ object SQSException { errorType = "com.amazonaws.sqs#QueueDoesNotExist", errorMessage = Some("The specified queue does not exist.") ) + + def resourceNotFoundException: SQSException = + new SQSException( + "AWS.SimpleQueueService.ResourceNotFoundException", + errorType = "com.amazonaws.sqs#ResourceNotFoundException", + errorMessage = Some("One or more specified resources don't exist.") + ) } diff --git a/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/SQSRestServerBuilder.scala b/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/SQSRestServerBuilder.scala index a3b726ca9..74267474a 100644 --- a/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/SQSRestServerBuilder.scala +++ b/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/SQSRestServerBuilder.scala @@ -167,7 +167,8 @@ case class TheSQSRestServerBuilder( with QueueAttributesOps with ListDeadLetterSourceQueuesDirectives with StartMessageMoveTaskDirectives - with CancelMessageMoveTaskDirectives { + with CancelMessageMoveTaskDirectives + with ListMessageMoveTasksDirectives { def serverAddress = currentServerAddress.get() @@ -210,6 +211,7 @@ case class TheSQSRestServerBuilder( listDeadLetterSourceQueues(p) ~ startMessageMoveTask(p) ~ cancelMessageMoveTask(p) ~ + listMessageMoveTasks(p) ~ // 4. Unmatched action unmatchedAction(p) diff --git a/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/StartMessageMoveTaskDirectives.scala b/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/StartMessageMoveTaskDirectives.scala index 4d96fc88a..3125b7fa2 100644 --- a/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/StartMessageMoveTaskDirectives.scala +++ b/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/StartMessageMoveTaskDirectives.scala @@ -12,48 +12,48 @@ import org.elasticmq.rest.sqs.model.RequestPayload import spray.json.DefaultJsonProtocol._ import spray.json.RootJsonFormat -import scala.async.Async._ - -trait StartMessageMoveTaskDirectives { this: ElasticMQDirectives with QueueURLModule with ResponseMarshaller => - - private val Arn = "(?:.+:(.+)?:(.+)?:)?(.+)".r +trait StartMessageMoveTaskDirectives extends ArnSupport { + this: ElasticMQDirectives with QueueURLModule with ResponseMarshaller => def startMessageMoveTask(p: RequestPayload)(implicit marshallerDependencies: MarshallerDependencies): Route = { p.action(StartMessageMoveTaskAction) { val params = p.as[StartMessageMoveTaskActionRequest] - val sourceQueueName = arnToQueueName(params.SourceArn) + val sourceQueueName = extractQueueName(params.SourceArn) queueActorAndDataFromQueueName(sourceQueueName) { (sourceQueue, _) => params.DestinationArn match { case Some(destinationQueueArn) => - val destinationQueueName = arnToQueueName(destinationQueueArn) + val destinationQueueName = extractQueueName(destinationQueueArn) queueActorAndDataFromQueueName(destinationQueueName) { (destinationQueue, _) => - startMessageMoveTask(sourceQueue, Some(destinationQueue), params.MaxNumberOfMessagesPerSecond) + startMessageMoveTask(sourceQueue, params.SourceArn, Some(destinationQueue), params.DestinationArn, params.MaxNumberOfMessagesPerSecond) } - case None => startMessageMoveTask(sourceQueue, None, params.MaxNumberOfMessagesPerSecond) + case None => startMessageMoveTask(sourceQueue, params.SourceArn, None, None, params.MaxNumberOfMessagesPerSecond) } } } } - private def arnToQueueName(arn: String): String = - arn match { - case Arn(_, _, queueName) => queueName - case _ => throw new SQSException("InvalidParameterValue") - } - private def startMessageMoveTask( sourceQueue: ActorRef, + sourceArn: String, destinationQueue: Option[ActorRef], + destinationArn: Option[String], maxNumberOfMessagesPerSecond: Option[Int] - )(implicit marshallerDependencies: MarshallerDependencies): Route = - async { - await( - queueManagerActor ? StartMessageMoveTask(sourceQueue, destinationQueue, maxNumberOfMessagesPerSecond) - ) match { + )(implicit marshallerDependencies: MarshallerDependencies): Route = { + for { + res <- queueManagerActor ? StartMessageMoveTask( + sourceQueue, + sourceArn, + destinationQueue, + destinationArn, + maxNumberOfMessagesPerSecond + ) + } yield { + res match { case Left(e: ElasticMQError) => throw new SQSException(e.code, errorMessage = Some(e.message)) case Right(taskHandle) => complete(StartMessageMoveTaskResponse(taskHandle)) } } + } } case class StartMessageMoveTaskActionRequest( From b852f8e9f42bf4ded14996dd182d27ce6a50b49f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Ossowski?= Date: Fri, 5 Apr 2024 10:14:16 +0200 Subject: [PATCH 06/12] fix scala 2.12 build --- .../rest/sqs/MessageMoveTaskTest.scala | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/MessageMoveTaskTest.scala b/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/MessageMoveTaskTest.scala index 1f792d7fe..a61b3a61c 100644 --- a/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/MessageMoveTaskTest.scala +++ b/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/MessageMoveTaskTest.scala @@ -1,7 +1,7 @@ package org.elasticmq.rest.sqs import com.amazonaws.services.sqs.model.QueueAttributeName.ApproximateNumberOfMessages -import com.amazonaws.services.sqs.model._ +import com.amazonaws.services.sqs.model.{CancelMessageMoveTaskRequest => AWSCancelMessageMoveTaskRequest, ListMessageMoveTasksRequest => AWSListMessageMoveTasksRequest, _} import org.elasticmq.rest.sqs.model.RedrivePolicy import org.elasticmq.rest.sqs.model.RedrivePolicyJson.format import org.scalatest.concurrent.Eventually @@ -93,7 +93,9 @@ class MessageMoveTaskTest extends SqsClientServerCommunication with Matchers wit // and val results = - client.listMessageMoveTasks(new ListMessageMoveTasksRequest().withSourceArn(DlqArn).withMaxResults(10)).getResults + client + .listMessageMoveTasks(new AWSListMessageMoveTasksRequest().withSourceArn(DlqArn).withMaxResults(10)) + .getResults // then results.size shouldEqual 1 @@ -127,7 +129,9 @@ class MessageMoveTaskTest extends SqsClientServerCommunication with Matchers wit // and val results = - client.listMessageMoveTasks(new ListMessageMoveTasksRequest().withSourceArn(DlqArn).withMaxResults(10)).getResults + client + .listMessageMoveTasks(new AWSListMessageMoveTasksRequest().withSourceArn(DlqArn).withMaxResults(10)) + .getResults // then results.size shouldEqual 2 @@ -140,7 +144,7 @@ class MessageMoveTaskTest extends SqsClientServerCommunication with Matchers wit test("should fail to list tasks for non-existing source ARN") { intercept[QueueDoesNotExistException] { client.listMessageMoveTasks( - new ListMessageMoveTasksRequest() + new AWSListMessageMoveTasksRequest() .withSourceArn(s"arn:aws:sqs:$awsRegion:$awsAccountId:nonExistingQueue") .withMaxResults(10) ) @@ -150,7 +154,7 @@ class MessageMoveTaskTest extends SqsClientServerCommunication with Matchers wit test("should fail to list tasks for invalid ARN") { intercept[QueueDoesNotExistException] { client.listMessageMoveTasks( - new ListMessageMoveTasksRequest() + new AWSListMessageMoveTasksRequest() .withSourceArn("invalidArn") .withMaxResults(10) ) @@ -173,7 +177,7 @@ class MessageMoveTaskTest extends SqsClientServerCommunication with Matchers wit // and: cancel the task after 2 seconds Thread.sleep(2000) val numMessagesMoved = client - .cancelMessageMoveTask(new CancelMessageMoveTaskRequest().withTaskHandle(taskHandle)) + .cancelMessageMoveTask(new AWSCancelMessageMoveTaskRequest().withTaskHandle(taskHandle)) .getApproximateNumberOfMessagesMoved // and: fetch ApproximateNumberOfMessages @@ -192,14 +196,14 @@ class MessageMoveTaskTest extends SqsClientServerCommunication with Matchers wit test("should fail to cancel non-existing task") { intercept[ResourceNotFoundException] { client - .cancelMessageMoveTask(new CancelMessageMoveTaskRequest().withTaskHandle(UUID.randomUUID().toString)) + .cancelMessageMoveTask(new AWSCancelMessageMoveTaskRequest().withTaskHandle(UUID.randomUUID().toString)) .getApproximateNumberOfMessagesMoved } } private def createQueuesAndPopulateDlq(): (CreateQueueResult, CreateQueueResult) = { val dlq = client.createQueue(new CreateQueueRequest("testQueue-dlq")) - val redrivePolicy = RedrivePolicy("testQueue-dlq", awsRegion, awsAccountId, 1).toJson.toString() + val redrivePolicy = RedrivePolicy("testQueue-dlq", awsRegion, awsAccountId, 1).toJson.compactPrint val queue = client.createQueue( new CreateQueueRequest("testQueue") From 8a8e04d8e40955252774ed3d3c47dfb7883b977d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Ossowski?= Date: Fri, 5 Apr 2024 10:58:03 +0200 Subject: [PATCH 07/12] refactor ListMessageMoveTasksDirectives --- .../sqs/ListMessageMoveTasksDirectives.scala | 41 +++++++++---------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/ListMessageMoveTasksDirectives.scala b/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/ListMessageMoveTasksDirectives.scala index 045fd8f48..81cd61b84 100644 --- a/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/ListMessageMoveTasksDirectives.scala +++ b/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/ListMessageMoveTasksDirectives.scala @@ -84,31 +84,28 @@ object ListMessageMoveTasksResponse { ListMessageMoveTasksResponse.apply ) - implicit val xmlSerializer: XmlSerializer[ListMessageMoveTasksResponse] = t => - - { - t.Results.map(task => - - {task.ApproximateNumberOfMessagesMoved} - { - task.ApproximateNumberOfMessagesToMove - } - {task.DestinationArn.map(arn => {arn}).getOrElse("")} - { - task.MaxNumberOfMessagesPerSecond - .map(v => {v}) - .getOrElse("") - } - {task.SourceArn} - {task.StartedTimestamp} - {task.Status} - {task.TaskHandle} - - ) - } + implicit val xmlSerializer: XmlSerializer[ListMessageMoveTasksResponse] = t => + + {t.Results.map(taskToEntry)} {EmptyRequestId} + + private def taskToEntry(task: MessageMoveTaskResponse) = + + {task.ApproximateNumberOfMessagesMoved} + {task.ApproximateNumberOfMessagesToMove} + {task.DestinationArn.map(arn => {arn}).getOrElse("")} + { + task.MaxNumberOfMessagesPerSecond + .map(v => {v}) + .getOrElse("") + } + {task.SourceArn} + {task.StartedTimestamp} + {task.Status} + {task.TaskHandle} + } From 8cef0dfecb2c579c3669e8c26833ed7ca78acc0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Ossowski?= Date: Fri, 5 Apr 2024 11:28:09 +0200 Subject: [PATCH 08/12] refactor QueueManagerActor --- .../elasticmq/actor/QueueManagerActor.scala | 82 ++--------------- .../actor/QueueManagerActorStorage.scala | 19 ++++ .../actor/QueueManagerMessageMoveOps.scala | 92 +++++++++++++++++++ 3 files changed, 120 insertions(+), 73 deletions(-) create mode 100644 core/src/main/scala/org/elasticmq/actor/QueueManagerActorStorage.scala create mode 100644 core/src/main/scala/org/elasticmq/actor/QueueManagerMessageMoveOps.scala diff --git a/core/src/main/scala/org/elasticmq/actor/QueueManagerActor.scala b/core/src/main/scala/org/elasticmq/actor/QueueManagerActor.scala index a2c8515a4..94899982e 100644 --- a/core/src/main/scala/org/elasticmq/actor/QueueManagerActor.scala +++ b/core/src/main/scala/org/elasticmq/actor/QueueManagerActor.scala @@ -1,31 +1,27 @@ package org.elasticmq.actor import org.apache.pekko.actor.{ActorRef, Props} -import org.apache.pekko.util.Timeout import org.elasticmq._ import org.elasticmq.actor.queue.{QueueActor, QueueEvent} import org.elasticmq.actor.reply._ import org.elasticmq.msg._ import org.elasticmq.util.{Logging, NowProvider} -import scala.concurrent.duration.DurationInt -import scala.concurrent.{ExecutionContext, Future} +import scala.collection.mutable import scala.reflect._ -import scala.util.{Failure, Success} class QueueManagerActor(nowProvider: NowProvider, limits: Limits, queueEventListener: Option[ActorRef]) extends ReplyingActor + with QueueManagerActorStorage + with QueueManagerMessageMoveOps with Logging { + type M[X] = QueueManagerMsg[X] val ev: ClassTag[QueueManagerMsg[Unit]] = classTag[M[Unit]] - implicit lazy val ec: ExecutionContext = context.dispatcher - implicit lazy val timeout: Timeout = 5.seconds - - case class ActorWithQueueData(actorRef: ActorRef, queueData: QueueData) - private val queues = collection.mutable.HashMap[String, ActorWithQueueData]() - private val messageMoveTasks = collection.mutable.HashMap[MessageMoveTaskId, ActorRef]() + val queues: mutable.Map[MessageMoveTaskId, ActorWithQueueData] = mutable.HashMap[String, ActorWithQueueData]() + // TODO: create *Ops class like in QueueActor def receiveAndReply[T](msg: QueueManagerMsg[T]): ReplyAction[T] = { val self = context.self msg match { @@ -80,69 +76,9 @@ class QueueManagerActor(nowProvider: NowProvider, limits: Limits, queueEventList destinationArn, maxNumberOfMessagesPerSecond ) => - val replyTo = sender() - val destination = destinationQueue.map(Future.successful).getOrElse { - val queueDataF = sourceQueue ? GetQueueData() - queueDataF.map { queueData => - queues - .filter { case (_, data) => - data.queueData.deadLettersQueue.exists(dlqData => dlqData.name == queueData.name) - } - .head - ._2 - .actorRef - } - } - destination - .flatMap(destinationQueueActorRef => { - val resultF = - sourceQueue ? StartMovingMessages( - destinationQueueActorRef, - destinationArn, - sourceArn, - maxNumberOfMessagesPerSecond, - self - ) - resultF.map(result => (result, destinationQueueActorRef)) - }) - .onComplete { - case Success((result, destinationQueueActorRef)) => - result match { - case Right(taskId) => - logger.debug("Message move task {} => {} created", sourceQueue, destinationQueueActorRef) - messageMoveTasks.put(taskId, sourceQueue) - replyTo ! Right(taskId) - case Left(error) => - logger.error("Failed to start message move task: {}", error) - replyTo ! Left(error) - } - case Failure(ex) => logger.error("Failed to start message move task", ex) - } - DoNotReply() - - case MessageMoveTaskFinished(taskHandle) => - logger.debug("Message move task {} finished", taskHandle) - messageMoveTasks.remove(taskHandle) - DoNotReply() - - case CancelMessageMoveTask(taskHandle) => - logger.info("Cancelling message move task {}", taskHandle) - messageMoveTasks.get(taskHandle) match { - case Some(sourceQueue) => - val replyTo = context.sender() - sourceQueue ? CancelMovingMessages() onComplete { - case Success(numMessageMoved) => - logger.debug("Message move task {} cancelled", taskHandle) - messageMoveTasks.remove(taskHandle) - replyTo ! Right(numMessageMoved) - case Failure(ex) => - logger.error("Failed to cancel message move task", ex) - replyTo ! Left(ex) - } - DoNotReply() - case None => - ReplyWith(Left(new InvalidMessageMoveTaskId(taskHandle))) - } + startMessageMoveTask(sourceQueue, sourceArn, destinationQueue, destinationArn, maxNumberOfMessagesPerSecond) + case MessageMoveTaskFinished(taskHandle) => onMessageMoveTaskFinished(taskHandle) + case CancelMessageMoveTask(taskHandle) => cancelMessageMoveTask(taskHandle) } } diff --git a/core/src/main/scala/org/elasticmq/actor/QueueManagerActorStorage.scala b/core/src/main/scala/org/elasticmq/actor/QueueManagerActorStorage.scala new file mode 100644 index 000000000..2392e683c --- /dev/null +++ b/core/src/main/scala/org/elasticmq/actor/QueueManagerActorStorage.scala @@ -0,0 +1,19 @@ +package org.elasticmq.actor +import org.apache.pekko.actor.{ActorContext, ActorRef} +import org.apache.pekko.util.Timeout +import org.elasticmq.QueueData + +import scala.collection.mutable +import scala.concurrent.ExecutionContext +import scala.concurrent.duration.DurationInt + +trait QueueManagerActorStorage { + + def context: ActorContext + + implicit lazy val ec: ExecutionContext = context.dispatcher + implicit lazy val timeout: Timeout = 5.seconds + + case class ActorWithQueueData(actorRef: ActorRef, queueData: QueueData) + def queues: mutable.Map[String, ActorWithQueueData] +} diff --git a/core/src/main/scala/org/elasticmq/actor/QueueManagerMessageMoveOps.scala b/core/src/main/scala/org/elasticmq/actor/QueueManagerMessageMoveOps.scala new file mode 100644 index 000000000..67432952a --- /dev/null +++ b/core/src/main/scala/org/elasticmq/actor/QueueManagerMessageMoveOps.scala @@ -0,0 +1,92 @@ +package org.elasticmq.actor +import org.apache.pekko.actor.ActorRef +import org.apache.pekko.util.Timeout +import org.elasticmq.actor.reply._ +import org.elasticmq.msg.{CancelMovingMessages, GetQueueData, MessageMoveTaskId, StartMovingMessages} +import org.elasticmq.util.Logging +import org.elasticmq.{ElasticMQError, InvalidMessageMoveTaskId} + +import scala.collection.mutable +import scala.concurrent.Future +import scala.util.{Failure, Success} + +trait QueueManagerMessageMoveOps extends Logging { + this: QueueManagerActorStorage => + + private val messageMoveTasks = mutable.HashMap[MessageMoveTaskId, ActorRef]() + + def startMessageMoveTask( + sourceQueue: ActorRef, + sourceArn: String, + destinationQueue: Option[ActorRef], + destinationArn: Option[String], + maxNumberOfMessagesPerSecond: Option[Int] + )(implicit timeout: Timeout): ReplyAction[Either[ElasticMQError, MessageMoveTaskId]] = { + val self = context.self + val replyTo = context.sender() + (for { + destinationQueueActorRef <- destinationQueue + .map(Future.successful) + .getOrElse(findDeadLetterQueueSource(sourceQueue)) + result <- sourceQueue ? StartMovingMessages( + destinationQueueActorRef, + destinationArn, + sourceArn, + maxNumberOfMessagesPerSecond, + self + ) + } yield (result, destinationQueueActorRef)).onComplete { + case Success((result, destinationQueueActorRef)) => + result match { + case Right(taskId) => + logger.debug("Message move task {} => {} created", sourceQueue, destinationQueueActorRef) + messageMoveTasks.put(taskId, sourceQueue) + replyTo ! Right(taskId) + case Left(error) => + logger.error("Failed to start message move task: {}", error) + replyTo ! Left(error) + } + case Failure(ex) => logger.error("Failed to start message move task", ex) + } + DoNotReply() + } + + def onMessageMoveTaskFinished(taskHandle: MessageMoveTaskId): ReplyAction[Unit] = { + logger.debug("Message move task {} finished", taskHandle) + messageMoveTasks.remove(taskHandle) + DoNotReply() + } + + def cancelMessageMoveTask(taskHandle: MessageMoveTaskId): ReplyAction[Either[ElasticMQError, Long]] = { + logger.info("Cancelling message move task {}", taskHandle) + messageMoveTasks.get(taskHandle) match { + case Some(sourceQueue) => + val replyTo = context.sender() + sourceQueue ? CancelMovingMessages() onComplete { + case Success(numMessageMoved) => + logger.debug("Message move task {} cancelled", taskHandle) + messageMoveTasks.remove(taskHandle) + replyTo ! Right(numMessageMoved) + case Failure(ex) => + logger.error("Failed to cancel message move task", ex) + replyTo ! Left(ex) + } + DoNotReply() + case None => + ReplyWith(Left(new InvalidMessageMoveTaskId(taskHandle))) + } + } + + private def findDeadLetterQueueSource(sourceQueue: ActorRef) = { + val queueDataF = sourceQueue ? GetQueueData() + queueDataF.map { queueData => + queues + .filter { case (_, data) => + data.queueData.deadLettersQueue.exists(dlqData => dlqData.name == queueData.name) + } + .head + ._2 + .actorRef + } + } +} From c01978417fec37df761c63d8c58f417aa1b4488d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Ossowski?= Date: Fri, 5 Apr 2024 11:35:08 +0200 Subject: [PATCH 09/12] refactor TaskId => TaskHandle --- .../scala/org/elasticmq/ElasticMQError.scala | 6 +++--- .../elasticmq/actor/QueueManagerActor.scala | 2 +- .../actor/QueueManagerMessageMoveOps.scala | 20 +++++++++---------- .../operations/MoveMessagesAsyncOps.scala | 8 ++++---- .../org/elasticmq/msg/QueueManagerMsg.scala | 6 +++--- .../scala/org/elasticmq/msg/QueueMsg.scala | 2 +- .../scala/org/elasticmq/msg/package.scala | 2 +- 7 files changed, 23 insertions(+), 23 deletions(-) diff --git a/core/src/main/scala/org/elasticmq/ElasticMQError.scala b/core/src/main/scala/org/elasticmq/ElasticMQError.scala index 1cef1ac94..855aa69c8 100644 --- a/core/src/main/scala/org/elasticmq/ElasticMQError.scala +++ b/core/src/main/scala/org/elasticmq/ElasticMQError.scala @@ -1,5 +1,5 @@ package org.elasticmq -import org.elasticmq.msg.MessageMoveTaskId +import org.elasticmq.msg.MessageMoveTaskHandle trait ElasticMQError { val queueName: String @@ -32,9 +32,9 @@ class InvalidReceiptHandle(val queueName: String, receiptHandle: String) extends val message = s"""The receipt handle "$receiptHandle" is not valid.""" } -class InvalidMessageMoveTaskId(val taskId: MessageMoveTaskId) extends ElasticMQError { +class InvalidMessageMoveTaskHandle(val taskHandle: MessageMoveTaskHandle) extends ElasticMQError { val code = "ResourceNotFoundException" - val message = s"""The task id "$taskId" is not valid or does not exist""" + val message = s"""The task handle "$taskHandle" is not valid or does not exist""" override val queueName: String = "invalid" } diff --git a/core/src/main/scala/org/elasticmq/actor/QueueManagerActor.scala b/core/src/main/scala/org/elasticmq/actor/QueueManagerActor.scala index 94899982e..934706927 100644 --- a/core/src/main/scala/org/elasticmq/actor/QueueManagerActor.scala +++ b/core/src/main/scala/org/elasticmq/actor/QueueManagerActor.scala @@ -19,7 +19,7 @@ class QueueManagerActor(nowProvider: NowProvider, limits: Limits, queueEventList type M[X] = QueueManagerMsg[X] val ev: ClassTag[QueueManagerMsg[Unit]] = classTag[M[Unit]] - val queues: mutable.Map[MessageMoveTaskId, ActorWithQueueData] = mutable.HashMap[String, ActorWithQueueData]() + val queues: mutable.Map[MessageMoveTaskHandle, ActorWithQueueData] = mutable.HashMap[String, ActorWithQueueData]() // TODO: create *Ops class like in QueueActor def receiveAndReply[T](msg: QueueManagerMsg[T]): ReplyAction[T] = { diff --git a/core/src/main/scala/org/elasticmq/actor/QueueManagerMessageMoveOps.scala b/core/src/main/scala/org/elasticmq/actor/QueueManagerMessageMoveOps.scala index 67432952a..9bac64e8a 100644 --- a/core/src/main/scala/org/elasticmq/actor/QueueManagerMessageMoveOps.scala +++ b/core/src/main/scala/org/elasticmq/actor/QueueManagerMessageMoveOps.scala @@ -2,9 +2,9 @@ package org.elasticmq.actor import org.apache.pekko.actor.ActorRef import org.apache.pekko.util.Timeout import org.elasticmq.actor.reply._ -import org.elasticmq.msg.{CancelMovingMessages, GetQueueData, MessageMoveTaskId, StartMovingMessages} +import org.elasticmq.msg.{CancelMovingMessages, GetQueueData, MessageMoveTaskHandle, StartMovingMessages} import org.elasticmq.util.Logging -import org.elasticmq.{ElasticMQError, InvalidMessageMoveTaskId} +import org.elasticmq.{ElasticMQError, InvalidMessageMoveTaskHandle} import scala.collection.mutable import scala.concurrent.Future @@ -13,7 +13,7 @@ import scala.util.{Failure, Success} trait QueueManagerMessageMoveOps extends Logging { this: QueueManagerActorStorage => - private val messageMoveTasks = mutable.HashMap[MessageMoveTaskId, ActorRef]() + private val messageMoveTasks = mutable.HashMap[MessageMoveTaskHandle, ActorRef]() def startMessageMoveTask( sourceQueue: ActorRef, @@ -21,7 +21,7 @@ trait QueueManagerMessageMoveOps extends Logging { destinationQueue: Option[ActorRef], destinationArn: Option[String], maxNumberOfMessagesPerSecond: Option[Int] - )(implicit timeout: Timeout): ReplyAction[Either[ElasticMQError, MessageMoveTaskId]] = { + )(implicit timeout: Timeout): ReplyAction[Either[ElasticMQError, MessageMoveTaskHandle]] = { val self = context.self val replyTo = context.sender() (for { @@ -38,10 +38,10 @@ trait QueueManagerMessageMoveOps extends Logging { } yield (result, destinationQueueActorRef)).onComplete { case Success((result, destinationQueueActorRef)) => result match { - case Right(taskId) => + case Right(taskHandle) => logger.debug("Message move task {} => {} created", sourceQueue, destinationQueueActorRef) - messageMoveTasks.put(taskId, sourceQueue) - replyTo ! Right(taskId) + messageMoveTasks.put(taskHandle, sourceQueue) + replyTo ! Right(taskHandle) case Left(error) => logger.error("Failed to start message move task: {}", error) replyTo ! Left(error) @@ -51,13 +51,13 @@ trait QueueManagerMessageMoveOps extends Logging { DoNotReply() } - def onMessageMoveTaskFinished(taskHandle: MessageMoveTaskId): ReplyAction[Unit] = { + def onMessageMoveTaskFinished(taskHandle: MessageMoveTaskHandle): ReplyAction[Unit] = { logger.debug("Message move task {} finished", taskHandle) messageMoveTasks.remove(taskHandle) DoNotReply() } - def cancelMessageMoveTask(taskHandle: MessageMoveTaskId): ReplyAction[Either[ElasticMQError, Long]] = { + def cancelMessageMoveTask(taskHandle: MessageMoveTaskHandle): ReplyAction[Either[ElasticMQError, Long]] = { logger.info("Cancelling message move task {}", taskHandle) messageMoveTasks.get(taskHandle) match { case Some(sourceQueue) => @@ -73,7 +73,7 @@ trait QueueManagerMessageMoveOps extends Logging { } DoNotReply() case None => - ReplyWith(Left(new InvalidMessageMoveTaskId(taskHandle))) + ReplyWith(Left(new InvalidMessageMoveTaskHandle(taskHandle))) } } diff --git a/core/src/main/scala/org/elasticmq/actor/queue/operations/MoveMessagesAsyncOps.scala b/core/src/main/scala/org/elasticmq/actor/queue/operations/MoveMessagesAsyncOps.scala index 73c946b55..30d8057fe 100644 --- a/core/src/main/scala/org/elasticmq/actor/queue/operations/MoveMessagesAsyncOps.scala +++ b/core/src/main/scala/org/elasticmq/actor/queue/operations/MoveMessagesAsyncOps.scala @@ -2,7 +2,7 @@ package org.elasticmq.actor.queue.operations import org.apache.pekko.actor.ActorRef import org.elasticmq.actor.queue.{QueueActorStorage, QueueEvent} -import org.elasticmq.msg.{MessageMoveTaskFinished, MessageMoveTaskId, MoveFirstMessage, SendMessage} +import org.elasticmq.msg.{MessageMoveTaskFinished, MessageMoveTaskHandle, MoveFirstMessage, SendMessage} import org.elasticmq.util.Logging import org.elasticmq.{ElasticMQError, MessageMoveTaskAlreadyRunning} @@ -18,7 +18,7 @@ case class MovingMessagesInProgress( maxNumberOfMessagesPerSecond: Option[Int], sourceArn: String, startedTimestamp: Long, - taskHandle: MessageMoveTaskId + taskHandle: MessageMoveTaskHandle ) extends MessageMoveTaskState case class MessageMoveTaskData( @@ -29,7 +29,7 @@ case class MessageMoveTaskData( sourceArn: String, startedTimestamp: Long, status: String, // RUNNING, COMPLETED, CANCELLING, CANCELLED, and FAILED - taskHandle: MessageMoveTaskId + taskHandle: MessageMoveTaskHandle ) trait MoveMessagesAsyncOps extends Logging { @@ -44,7 +44,7 @@ trait MoveMessagesAsyncOps extends Logging { sourceArn: String, maxNumberOfMessagesPerSecond: Option[Int], queueManager: ActorRef - ): Either[ElasticMQError, MessageMoveTaskId] = { + ): Either[ElasticMQError, MessageMoveTaskHandle] = { messageMoveTaskState match { case NotMovingMessages => val taskHandle = UUID.randomUUID().toString diff --git a/core/src/main/scala/org/elasticmq/msg/QueueManagerMsg.scala b/core/src/main/scala/org/elasticmq/msg/QueueManagerMsg.scala index 1bb7501f4..bfcbe837a 100644 --- a/core/src/main/scala/org/elasticmq/msg/QueueManagerMsg.scala +++ b/core/src/main/scala/org/elasticmq/msg/QueueManagerMsg.scala @@ -17,10 +17,10 @@ case class StartMessageMoveTask( destinationQueue: Option[ActorRef], destinationArn: Option[String], maxNumberOfMessagesPerSecond: Option[Int] -) extends QueueManagerMsg[Either[ElasticMQError, MessageMoveTaskId]] +) extends QueueManagerMsg[Either[ElasticMQError, MessageMoveTaskHandle]] case class MessageMoveTaskFinished( - taskHandle: MessageMoveTaskId + taskHandle: MessageMoveTaskHandle ) extends QueueManagerMsg[Unit] case class CancelMessageMoveTask( - taskHandle: MessageMoveTaskId + taskHandle: MessageMoveTaskHandle ) extends QueueManagerMsg[Either[ElasticMQError, Long]] diff --git a/core/src/main/scala/org/elasticmq/msg/QueueMsg.scala b/core/src/main/scala/org/elasticmq/msg/QueueMsg.scala index faf3d8a54..a8a8bf0f1 100644 --- a/core/src/main/scala/org/elasticmq/msg/QueueMsg.scala +++ b/core/src/main/scala/org/elasticmq/msg/QueueMsg.scala @@ -61,7 +61,7 @@ case class StartMovingMessages( sourceArn: String, maxNumberOfMessagesPerSecond: Option[Int], queueManager: ActorRef -) extends QueueMessageMsg[Either[ElasticMQError, MessageMoveTaskId]] +) extends QueueMessageMsg[Either[ElasticMQError, MessageMoveTaskHandle]] case class CancelMovingMessages() extends QueueMessageMsg[Long] case class MoveFirstMessage( destinationQueue: ActorRef, diff --git a/core/src/main/scala/org/elasticmq/msg/package.scala b/core/src/main/scala/org/elasticmq/msg/package.scala index 421fbffa4..310c97d6c 100644 --- a/core/src/main/scala/org/elasticmq/msg/package.scala +++ b/core/src/main/scala/org/elasticmq/msg/package.scala @@ -1,5 +1,5 @@ package org.elasticmq package object msg { - type MessageMoveTaskId = String + type MessageMoveTaskHandle = String } From 3b4b20226852e2448f645fb9a6cbad0a0fa0dc1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Ossowski?= Date: Mon, 8 Apr 2024 13:37:24 +0200 Subject: [PATCH 10/12] make MessageMoveTaskTest support both SDKs --- .../org/elasticmq/rest/sqs/AwsConfig.scala | 7 + .../rest/sqs/MessageMoveTaskTest.scala | 189 +++++++----------- .../sqs/SqsClientServerCommunication.scala | 7 +- ...qsClientServerWithSdkV2Communication.scala | 17 +- .../rest/sqs/client/AwsSdkV1SqsClient.scala | 114 +++++++++++ .../rest/sqs/client/AwsSdkV2SqsClient.scala | 124 ++++++++++++ .../rest/sqs/client/HasSqsTestClient.scala | 6 + .../elasticmq/rest/sqs/client/SqsClient.scala | 12 ++ .../rest/sqs/client/SqsClientError.scala | 3 + .../elasticmq/rest/sqs/client/package.scala | 52 +++++ .../sqs/CancelMessageMoveTaskDirectives.scala | 3 +- .../org/elasticmq/rest/sqs/SQSException.scala | 17 +- .../sqs/StartMessageMoveTaskDirectives.scala | 14 +- 13 files changed, 432 insertions(+), 133 deletions(-) create mode 100644 rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/AwsConfig.scala create mode 100644 rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/client/AwsSdkV1SqsClient.scala create mode 100644 rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/client/AwsSdkV2SqsClient.scala create mode 100644 rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/client/HasSqsTestClient.scala create mode 100644 rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/client/SqsClient.scala create mode 100644 rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/client/SqsClientError.scala create mode 100644 rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/client/package.scala diff --git a/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/AwsConfig.scala b/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/AwsConfig.scala new file mode 100644 index 000000000..9bfdec729 --- /dev/null +++ b/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/AwsConfig.scala @@ -0,0 +1,7 @@ +package org.elasticmq.rest.sqs + +trait AwsConfig { + + def awsRegion: String + def awsAccountId: String +} diff --git a/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/MessageMoveTaskTest.scala b/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/MessageMoveTaskTest.scala index a61b3a61c..91ce88197 100644 --- a/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/MessageMoveTaskTest.scala +++ b/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/MessageMoveTaskTest.scala @@ -1,17 +1,22 @@ package org.elasticmq.rest.sqs -import com.amazonaws.services.sqs.model.QueueAttributeName.ApproximateNumberOfMessages -import com.amazonaws.services.sqs.model.{CancelMessageMoveTaskRequest => AWSCancelMessageMoveTaskRequest, ListMessageMoveTasksRequest => AWSListMessageMoveTasksRequest, _} +import org.elasticmq.rest.sqs.client._ import org.elasticmq.rest.sqs.model.RedrivePolicy import org.elasticmq.rest.sqs.model.RedrivePolicyJson.format import org.scalatest.concurrent.Eventually +import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers import spray.json.enrichAny import java.util.UUID import scala.concurrent.duration.DurationInt -class MessageMoveTaskTest extends SqsClientServerCommunication with Matchers with Eventually { +abstract class MessageMoveTaskTest + extends AnyFunSuite + with HasSqsTestClient + with AwsConfig + with Matchers + with Eventually { private val NumMessages = 6 private val DlqArn = s"arn:aws:sqs:$awsRegion:$awsAccountId:testQueue-dlq" @@ -21,14 +26,12 @@ class MessageMoveTaskTest extends SqsClientServerCommunication with Matchers wit val (queue, dlq) = createQueuesAndPopulateDlq() // when: start message move task - client.startMessageMoveTask( - new StartMessageMoveTaskRequest().withSourceArn(DlqArn) - ) + testClient.startMessageMoveTask(DlqArn) // then: ensure that messages are moved back to the original queue eventually(timeout(5.seconds), interval(100.millis)) { - fetchApproximateNumberOfMessages(queue.getQueueUrl) shouldEqual NumMessages - fetchApproximateNumberOfMessages(dlq.getQueueUrl) shouldEqual 0 + fetchApproximateNumberOfMessages(queue) shouldEqual NumMessages + fetchApproximateNumberOfMessages(dlq) shouldEqual 0 } } @@ -37,16 +40,12 @@ class MessageMoveTaskTest extends SqsClientServerCommunication with Matchers wit val (queue, dlq) = createQueuesAndPopulateDlq() // when: start message move task - client.startMessageMoveTask( - new StartMessageMoveTaskRequest() - .withSourceArn(DlqArn) - .withMaxNumberOfMessagesPerSecond(1) - ) + testClient.startMessageMoveTask(DlqArn, maxNumberOfMessagesPerSecond = Some(1)) // then: ensure that not all messages were moved back to the original queue after 2 seconds Thread.sleep(2000) - fetchApproximateNumberOfMessages(queue.getQueueUrl) should (be > 1 and be < 6) - fetchApproximateNumberOfMessages(dlq.getQueueUrl) should (be > 1 and be < 6) + fetchApproximateNumberOfMessages(queue) should (be > 1 and be < 6) + fetchApproximateNumberOfMessages(dlq) should (be > 1 and be < 6) } test("should not run two message move tasks in parallel") { @@ -54,28 +53,23 @@ class MessageMoveTaskTest extends SqsClientServerCommunication with Matchers wit val (queue, dlq) = createQueuesAndPopulateDlq() // when: start message move task - client.startMessageMoveTask( - new StartMessageMoveTaskRequest() - .withSourceArn(DlqArn) - .withMaxNumberOfMessagesPerSecond(1) - ) + testClient.startMessageMoveTask(DlqArn, maxNumberOfMessagesPerSecond = Some(1)) // and: try to start another message move task - val thrown = intercept[UnsupportedOperationException] { - client.startMessageMoveTask( - new StartMessageMoveTaskRequest() - .withSourceArn(DlqArn) - .withMaxNumberOfMessagesPerSecond(1) - ) - } + val result = testClient.startMessageMoveTask(DlqArn) // then - thrown.getErrorMessage shouldBe "A message move task is already running on queue \"testQueue-dlq\"" + result shouldBe Left( + SqsClientError( + UnsupportedOperation, + "A message move task is already running on queue \"testQueue-dlq\"" + ) + ) // and: ensure that not all messages were moved back to the original queue after 2 seconds Thread.sleep(2000) - fetchApproximateNumberOfMessages(queue.getQueueUrl) should (be > 1 and be < 6) - fetchApproximateNumberOfMessages(dlq.getQueueUrl) should (be > 1 and be < 6) + fetchApproximateNumberOfMessages(queue) should (be > 1 and be < 6) + fetchApproximateNumberOfMessages(dlq) should (be > 1 and be < 6) } test("should run message move task and list it") { @@ -83,24 +77,15 @@ class MessageMoveTaskTest extends SqsClientServerCommunication with Matchers wit val (queue, dlq) = createQueuesAndPopulateDlq() // when - val taskHandle = client - .startMessageMoveTask( - new StartMessageMoveTaskRequest() - .withSourceArn(DlqArn) - .withMaxNumberOfMessagesPerSecond(1) - ) - .getTaskHandle + val taskHandle = testClient.startMessageMoveTask(DlqArn, maxNumberOfMessagesPerSecond = Some(1)).right.get // and - val results = - client - .listMessageMoveTasks(new AWSListMessageMoveTasksRequest().withSourceArn(DlqArn).withMaxResults(10)) - .getResults + val results = testClient.listMessageMoveTasks(DlqArn, maxResults = Some(10)).right.get // then results.size shouldEqual 1 - results.get(0).getTaskHandle shouldBe taskHandle - results.get(0).getStatus shouldBe "RUNNING" + results.head.taskHandle shouldBe taskHandle + results.head.status shouldBe "RUNNING" } test("should run two message move task and list them in the correct order") { @@ -108,57 +93,35 @@ class MessageMoveTaskTest extends SqsClientServerCommunication with Matchers wit val (queue, dlq) = createQueuesAndPopulateDlq() // when - val firstTaskHandle = client - .startMessageMoveTask( - new StartMessageMoveTaskRequest() - .withSourceArn(DlqArn) - ) - .getTaskHandle + val firstTaskHandle = testClient.startMessageMoveTask(DlqArn).right.get // and receiveAllMessagesTwice(queue) // and - val secondTaskHandle = client - .startMessageMoveTask( - new StartMessageMoveTaskRequest() - .withSourceArn(DlqArn) - .withMaxNumberOfMessagesPerSecond(1) - ) - .getTaskHandle + val secondTaskHandle = testClient.startMessageMoveTask(DlqArn, maxNumberOfMessagesPerSecond = Some(1)).right.get // and - val results = - client - .listMessageMoveTasks(new AWSListMessageMoveTasksRequest().withSourceArn(DlqArn).withMaxResults(10)) - .getResults + val results = testClient.listMessageMoveTasks(DlqArn, maxResults = Some(10)).right.get // then results.size shouldEqual 2 - results.get(0).getTaskHandle shouldBe secondTaskHandle - results.get(0).getStatus shouldBe "RUNNING" - results.get(1).getTaskHandle shouldBe firstTaskHandle - results.get(1).getStatus shouldBe "COMPLETED" + results(0).taskHandle shouldBe secondTaskHandle + results(0).status shouldBe "RUNNING" + results(1).taskHandle shouldBe firstTaskHandle + results(1).status shouldBe "COMPLETED" } test("should fail to list tasks for non-existing source ARN") { - intercept[QueueDoesNotExistException] { - client.listMessageMoveTasks( - new AWSListMessageMoveTasksRequest() - .withSourceArn(s"arn:aws:sqs:$awsRegion:$awsAccountId:nonExistingQueue") - .withMaxResults(10) - ) - } + testClient.listMessageMoveTasks(s"arn:aws:sqs:$awsRegion:$awsAccountId:nonExistingQueue") shouldBe Left( + SqsClientError(QueueDoesNotExist, "The specified queue does not exist.") + ) } test("should fail to list tasks for invalid ARN") { - intercept[QueueDoesNotExistException] { - client.listMessageMoveTasks( - new AWSListMessageMoveTasksRequest() - .withSourceArn("invalidArn") - .withMaxResults(10) - ) - } + testClient.listMessageMoveTasks(s"invalidArn") shouldBe Left( + SqsClientError(QueueDoesNotExist, "The specified queue does not exist.") + ) } test("should run and cancel message move task") { @@ -166,54 +129,47 @@ class MessageMoveTaskTest extends SqsClientServerCommunication with Matchers wit val (queue, dlq) = createQueuesAndPopulateDlq() // when: start message move task - val taskHandle = client - .startMessageMoveTask( - new StartMessageMoveTaskRequest() - .withSourceArn(DlqArn) - .withMaxNumberOfMessagesPerSecond(1) - ) - .getTaskHandle + val taskHandle = testClient.startMessageMoveTask(DlqArn, maxNumberOfMessagesPerSecond = Some(1)).right.get // and: cancel the task after 2 seconds Thread.sleep(2000) - val numMessagesMoved = client - .cancelMessageMoveTask(new AWSCancelMessageMoveTaskRequest().withTaskHandle(taskHandle)) - .getApproximateNumberOfMessagesMoved + val numMessagesMoved = testClient.cancelMessageMoveTask(taskHandle).right.get // and: fetch ApproximateNumberOfMessages - val numMessagesInMainQueue = fetchApproximateNumberOfMessages(queue.getQueueUrl) - val numMessagesInDlQueue = fetchApproximateNumberOfMessages(dlq.getQueueUrl) + val numMessagesInMainQueue = fetchApproximateNumberOfMessages(queue) + val numMessagesInDlQueue = fetchApproximateNumberOfMessages(dlq) // then numMessagesMoved shouldEqual numMessagesInMainQueue // then: ApproximateNumberOfMessages should not change after 2 seconds Thread.sleep(2000) - fetchApproximateNumberOfMessages(queue.getQueueUrl) shouldEqual numMessagesInMainQueue - fetchApproximateNumberOfMessages(dlq.getQueueUrl) shouldEqual numMessagesInDlQueue + fetchApproximateNumberOfMessages(queue) shouldEqual numMessagesInMainQueue + fetchApproximateNumberOfMessages(dlq) shouldEqual numMessagesInDlQueue } test("should fail to cancel non-existing task") { - intercept[ResourceNotFoundException] { - client - .cancelMessageMoveTask(new AWSCancelMessageMoveTaskRequest().withTaskHandle(UUID.randomUUID().toString)) - .getApproximateNumberOfMessagesMoved - } + val randomTaskHandle = UUID.randomUUID().toString + testClient.cancelMessageMoveTask(randomTaskHandle) shouldBe Left( + SqsClientError(ResourceNotFound, s"The task handle ${'"'}$randomTaskHandle${'"'} is not valid or does not exist") + ) } - private def createQueuesAndPopulateDlq(): (CreateQueueResult, CreateQueueResult) = { - val dlq = client.createQueue(new CreateQueueRequest("testQueue-dlq")) + private def createQueuesAndPopulateDlq(): (QueueUrl, QueueUrl) = { + val dlq = testClient.createQueue("testQueue-dlq") val redrivePolicy = RedrivePolicy("testQueue-dlq", awsRegion, awsAccountId, 1).toJson.compactPrint val queue = - client.createQueue( - new CreateQueueRequest("testQueue") - .addAttributesEntry(redrivePolicyAttribute, redrivePolicy) - .addAttributesEntry("VisibilityTimeout", "1") + testClient.createQueue( + "testQueue", + attributes = Map( + RedrivePolicyAttributeName -> redrivePolicy, + VisibilityTimeoutAttributeName -> "1" + ) ) // when: send messages for (i <- 0 until NumMessages) { - client.sendMessage(queue.getQueueUrl, "Test message " + i) + testClient.sendMessage(queue, "Test message " + i) } // and: receive messages twice to make them move to DLQ @@ -221,31 +177,32 @@ class MessageMoveTaskTest extends SqsClientServerCommunication with Matchers wit // then: ensure that messages are in DLQ eventually(timeout(2.seconds), interval(100.millis)) { - fetchApproximateNumberOfMessages(queue.getQueueUrl) shouldEqual 0 - fetchApproximateNumberOfMessages(dlq.getQueueUrl) shouldEqual NumMessages + fetchApproximateNumberOfMessages(queue) shouldEqual 0 + fetchApproximateNumberOfMessages(dlq) shouldEqual NumMessages } (queue, dlq) } - private def receiveAllMessagesTwice(queue: CreateQueueResult): Unit = { - for (i <- 0 until NumMessages) { - client.receiveMessage(queue.getQueueUrl) + private def receiveAllMessagesTwice(queue: QueueUrl): Unit = { + for (_ <- 0 until NumMessages) { + testClient.receiveMessage(queue) } Thread.sleep(1500) - for (i <- 0 until NumMessages) { - client.receiveMessage(queue.getQueueUrl) + for (_ <- 0 until NumMessages) { + testClient.receiveMessage(queue) } } private def fetchApproximateNumberOfMessages(queueUrl: String): Int = { - client - .getQueueAttributes( - new GetQueueAttributesRequest().withQueueUrl(queueUrl).withAttributeNames(ApproximateNumberOfMessages) + testClient + .getQueueAttributes(queueUrl, ApproximateNumberOfMessagesAttributeName)( + ApproximateNumberOfMessagesAttributeName.value ) - .getAttributes - .get(ApproximateNumberOfMessages.toString) .toInt } } + +class MessageMoveTaskSdkV1Test extends MessageMoveTaskTest with SqsClientServerCommunication +class MessageMoveTaskSdkV2Test extends MessageMoveTaskTest with SqsClientServerWithSdkV2Communication diff --git a/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/SqsClientServerCommunication.scala b/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/SqsClientServerCommunication.scala index 423bfe1a0..3a404c57c 100644 --- a/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/SqsClientServerCommunication.scala +++ b/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/SqsClientServerCommunication.scala @@ -5,16 +5,19 @@ import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration import com.amazonaws.services.sqs.model.{Message, ReceiveMessageRequest} import com.amazonaws.services.sqs.{AmazonSQS, AmazonSQSClientBuilder} import org.apache.http.impl.client.{CloseableHttpClient, HttpClients} +import org.elasticmq.rest.sqs.client.{AwsSdkV1SqsClient, SqsClient} import org.elasticmq.util.Logging import org.elasticmq.{NodeAddress, RelaxedSQSLimits} -import org.scalatest.{Args, BeforeAndAfter, Status} import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.{Args, BeforeAndAfter, Status} import scala.collection.JavaConverters._ import scala.util.Try trait SqsClientServerCommunication extends AnyFunSuite with BeforeAndAfter with Logging { + var testClient: SqsClient = _ + var client: AmazonSQS = _ // strict server var relaxedClient: AmazonSQS = _ var httpClient: CloseableHttpClient = _ @@ -55,6 +58,8 @@ trait SqsClientServerCommunication extends AnyFunSuite with BeforeAndAfter with .withEndpointConfiguration(new EndpointConfiguration(ServiceEndpoint, "us-east-1")) .build() + testClient = new AwsSdkV1SqsClient(client) + relaxedClient = AmazonSQSClientBuilder .standard() .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials("x", "x"))) diff --git a/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/SqsClientServerWithSdkV2Communication.scala b/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/SqsClientServerWithSdkV2Communication.scala index 217f0427c..9223e9eec 100644 --- a/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/SqsClientServerWithSdkV2Communication.scala +++ b/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/SqsClientServerWithSdkV2Communication.scala @@ -1,20 +1,23 @@ package org.elasticmq.rest.sqs +import org.elasticmq.rest.sqs.client.{AwsSdkV2SqsClient, SqsClient} import org.elasticmq.util.Logging import org.elasticmq.{NodeAddress, RelaxedSQSLimits} +import org.scalatest.BeforeAndAfter import org.scalatest.funsuite.AnyFunSuite -import org.scalatest.{Args, BeforeAndAfter, Status} import software.amazon.awssdk.auth.credentials.{AwsBasicCredentials, StaticCredentialsProvider} import software.amazon.awssdk.regions.Region -import software.amazon.awssdk.services.sqs.{SqsClient, SqsClientBuilder} +import software.amazon.awssdk.services.sqs.{SqsClient => AwsSqsClient} import java.net.URI import scala.util.Try trait SqsClientServerWithSdkV2Communication extends AnyFunSuite with BeforeAndAfter with Logging { - var clientV2: SqsClient = _ // strict server - var relaxedClientV2: SqsClient = _ + var testClient: SqsClient = _ + + var clientV2: AwsSqsClient = _ // strict server + var relaxedClientV2: AwsSqsClient = _ var currentTestName: String = _ @@ -42,14 +45,16 @@ trait SqsClientServerWithSdkV2Communication extends AnyFunSuite with BeforeAndAf strictServer.waitUntilStarted() relaxedServer.waitUntilStarted() - clientV2 = SqsClient + clientV2 = AwsSqsClient .builder() .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create("x", "x"))) .region(Region.EU_CENTRAL_1) .endpointOverride(new URI("http://localhost:9321")) .build() - relaxedClientV2 = SqsClient + testClient = new AwsSdkV2SqsClient(clientV2) + + relaxedClientV2 = AwsSqsClient .builder() .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create("x", "x"))) .region(Region.EU_CENTRAL_1) diff --git a/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/client/AwsSdkV1SqsClient.scala b/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/client/AwsSdkV1SqsClient.scala new file mode 100644 index 000000000..5a81d2e4f --- /dev/null +++ b/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/client/AwsSdkV1SqsClient.scala @@ -0,0 +1,114 @@ +package org.elasticmq.rest.sqs.client + +import com.amazonaws.services.sqs.AmazonSQS +import com.amazonaws.services.sqs.model.{CancelMessageMoveTaskRequest, CreateQueueRequest, GetQueueAttributesRequest, ListMessageMoveTasksRequest, QueueDoesNotExistException, ReceiveMessageRequest, ResourceNotFoundException, SendMessageRequest, StartMessageMoveTaskRequest, UnsupportedOperationException} + +import scala.jdk.CollectionConverters.{iterableAsScalaIterableConverter, mapAsJavaMapConverter, mapAsScalaMapConverter, seqAsJavaListConverter} + +class AwsSdkV1SqsClient(client: AmazonSQS) extends SqsClient { + + override def createQueue( + queueName: String, + attributes: Map[ + QueueAttributeName, + String + ] = Map.empty + ): QueueUrl = client + .createQueue( + new CreateQueueRequest() + .withQueueName(queueName) + .withAttributes(attributes.map { case (k, v) => (k.value, v) }.asJava) + ) + .getQueueUrl + + override def sendMessage( + queueUrl: QueueUrl, + messageBody: String + ): Unit = client.sendMessage( + new SendMessageRequest() + .withQueueUrl(queueUrl) + .withMessageBody(messageBody) + ) + + override def receiveMessage(queueUrl: QueueUrl): List[ReceivedMessage] = + client.receiveMessage(new ReceiveMessageRequest().withQueueUrl(queueUrl)).getMessages.asScala.toList.map { msg => + ReceivedMessage(msg.getMessageId, msg.getReceiptHandle, msg.getBody) + } + + override def getQueueAttributes( + queueUrl: QueueUrl, + attributeNames: QueueAttributeName* + ): Map[String, String] = client + .getQueueAttributes( + new GetQueueAttributesRequest() + .withQueueUrl(queueUrl) + .withAttributeNames(attributeNames.toList.map(_.value).asJava) + ) + .getAttributes + .asScala + .toMap + + override def startMessageMoveTask( + sourceArn: Arn, + maxNumberOfMessagesPerSecond: Option[Int] + ): Either[SqsClientError, TaskHandle] = interceptErrors { + client + .startMessageMoveTask( + new StartMessageMoveTaskRequest() + .withSourceArn(sourceArn) + .withMaxNumberOfMessagesPerSecond(maxNumberOfMessagesPerSecond match { + case Some(value) => value + case None => null + }) + ) + .getTaskHandle + } + + override def listMessageMoveTasks( + sourceArn: Arn, + maxResults: Option[Int] + ): Either[SqsClientError, List[MessageMoveTask]] = interceptErrors { + client + .listMessageMoveTasks( + new ListMessageMoveTasksRequest() + .withSourceArn(sourceArn) + .withMaxResults(maxResults match { + case Some(value) => value + case None => null + }) + ) + .getResults + .asScala + .toList + .map { task => + MessageMoveTask( + task.getTaskHandle, + task.getSourceArn, + task.getStatus, + Option(task.getMaxNumberOfMessagesPerSecond).map(_.intValue()) + ) + } + } + + override def cancelMessageMoveTask( + taskHandle: TaskHandle + ): Either[SqsClientError, ApproximateNumberOfMessagesMoved] = interceptErrors { + client + .cancelMessageMoveTask(new CancelMessageMoveTaskRequest().withTaskHandle(taskHandle)) + .getApproximateNumberOfMessagesMoved + } + + private def interceptErrors[T](f: => T): Either[SqsClientError, T] = { + try { + Right(f) + } catch { + case e: UnsupportedOperationException => + Left(SqsClientError(UnsupportedOperation, e.getErrorMessage)) + case e: ResourceNotFoundException => + Left(SqsClientError(ResourceNotFound, e.getErrorMessage)) + case e: QueueDoesNotExistException => + Left(SqsClientError(QueueDoesNotExist, e.getErrorMessage)) + case e: Exception => Left(SqsClientError(UnknownSqsClientErrorType, e.getMessage)) + } + } +} diff --git a/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/client/AwsSdkV2SqsClient.scala b/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/client/AwsSdkV2SqsClient.scala new file mode 100644 index 000000000..22d6cffb6 --- /dev/null +++ b/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/client/AwsSdkV2SqsClient.scala @@ -0,0 +1,124 @@ +package org.elasticmq.rest.sqs.client +import software.amazon.awssdk.services.sqs.model.{CancelMessageMoveTaskRequest, CreateQueueRequest, GetQueueAttributesRequest, ListMessageMoveTasksRequest, QueueDoesNotExistException, ReceiveMessageRequest, ResourceNotFoundException, SendMessageRequest, StartMessageMoveTaskRequest, UnsupportedOperationException, QueueAttributeName => AwsQueueAttributeName} + +import scala.jdk.CollectionConverters.{iterableAsScalaIterableConverter, mapAsJavaMapConverter, mapAsScalaMapConverter, seqAsJavaListConverter} + +class AwsSdkV2SqsClient(client: software.amazon.awssdk.services.sqs.SqsClient) extends SqsClient { + + override def createQueue( + queueName: String, + attributes: Map[ + QueueAttributeName, + String + ] = Map.empty + ): QueueUrl = client + .createQueue( + CreateQueueRequest + .builder() + .queueName(queueName) + .attributes(attributes.map { case (k, v) => (AwsQueueAttributeName.fromValue(k.value), v) }.asJava) + .build() + ) + .queueUrl() + + override def sendMessage( + queueUrl: QueueUrl, + messageBody: String + ): Unit = client.sendMessage( + SendMessageRequest + .builder() + .queueUrl(queueUrl) + .messageBody(messageBody) + .build() + ) + + override def receiveMessage(queueUrl: QueueUrl): List[ReceivedMessage] = + client.receiveMessage(ReceiveMessageRequest.builder().queueUrl(queueUrl).build()).messages().asScala.toList.map { + msg => + ReceivedMessage(msg.messageId(), msg.receiptHandle(), msg.body()) + } + + override def getQueueAttributes( + queueUrl: QueueUrl, + attributeNames: QueueAttributeName* + ): Map[String, String] = client + .getQueueAttributes( + GetQueueAttributesRequest + .builder() + .queueUrl(queueUrl) + .attributeNames(attributeNames.toList.map(atr => AwsQueueAttributeName.fromValue(atr.value)).asJava) + .build() + ) + .attributes() + .asScala + .map { case (k, v) => (k.toString, v) } + .toMap + + override def startMessageMoveTask( + sourceArn: Arn, + maxNumberOfMessagesPerSecond: Option[Int] + ): Either[SqsClientError, TaskHandle] = interceptErrors { + client + .startMessageMoveTask( + StartMessageMoveTaskRequest + .builder() + .sourceArn(sourceArn) + .maxNumberOfMessagesPerSecond(maxNumberOfMessagesPerSecond match { + case Some(value) => value + case None => null + }) + .build() + ) + .taskHandle() + } + + override def listMessageMoveTasks( + sourceArn: Arn, + maxResults: Option[Int] + ): Either[SqsClientError, List[MessageMoveTask]] = interceptErrors { + client + .listMessageMoveTasks( + ListMessageMoveTasksRequest + .builder() + .sourceArn(sourceArn) + .maxResults(maxResults match { + case Some(value) => value + case None => null + }) + .build() + ) + .results() + .asScala + .toList + .map { task => + MessageMoveTask( + task.taskHandle(), + task.sourceArn(), + task.status().toString, + Option(task.maxNumberOfMessagesPerSecond()).map(_.intValue()) + ) + } + } + + override def cancelMessageMoveTask( + taskHandle: TaskHandle + ): Either[SqsClientError, ApproximateNumberOfMessagesMoved] = interceptErrors { + client + .cancelMessageMoveTask(CancelMessageMoveTaskRequest.builder().taskHandle(taskHandle).build()) + .approximateNumberOfMessagesMoved() + } + + private def interceptErrors[T](f: => T): Either[SqsClientError, T] = { + try { + Right(f) + } catch { + case e: UnsupportedOperationException => + Left(SqsClientError(UnsupportedOperation, e.awsErrorDetails().errorMessage())) + case e: ResourceNotFoundException => + Left(SqsClientError(ResourceNotFound, e.awsErrorDetails().errorMessage())) + case e: QueueDoesNotExistException => + Left(SqsClientError(QueueDoesNotExist, e.awsErrorDetails().errorMessage())) + case e: Exception => Left(SqsClientError(UnknownSqsClientErrorType, e.getMessage)) + } + } +} diff --git a/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/client/HasSqsTestClient.scala b/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/client/HasSqsTestClient.scala new file mode 100644 index 000000000..6e9792974 --- /dev/null +++ b/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/client/HasSqsTestClient.scala @@ -0,0 +1,6 @@ +package org.elasticmq.rest.sqs.client + +trait HasSqsTestClient { + + def testClient: SqsClient +} diff --git a/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/client/SqsClient.scala b/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/client/SqsClient.scala new file mode 100644 index 000000000..7fb188b56 --- /dev/null +++ b/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/client/SqsClient.scala @@ -0,0 +1,12 @@ +package org.elasticmq.rest.sqs.client + +trait SqsClient { + + def createQueue(queueName: String, attributes: Map[QueueAttributeName, String] = Map.empty): QueueUrl + def sendMessage(queueUrl: QueueUrl, messageBody: String): Unit + def receiveMessage(queueUrl: QueueUrl): List[ReceivedMessage] + def getQueueAttributes(queueUrl: QueueUrl, attributeNames: QueueAttributeName*): Map[String, String] + def startMessageMoveTask(sourceArn: Arn, maxNumberOfMessagesPerSecond: Option[Int] = None): Either[SqsClientError, TaskHandle] + def listMessageMoveTasks(sourceArn: Arn, maxResults: Option[Int] = None): Either[SqsClientError, List[MessageMoveTask]] + def cancelMessageMoveTask(taskHandle: TaskHandle): Either[SqsClientError, ApproximateNumberOfMessagesMoved] +} diff --git a/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/client/SqsClientError.scala b/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/client/SqsClientError.scala new file mode 100644 index 000000000..7e6413b75 --- /dev/null +++ b/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/client/SqsClientError.scala @@ -0,0 +1,3 @@ +package org.elasticmq.rest.sqs.client + +case class SqsClientError(errorType: SqsClientErrorType, message: String) diff --git a/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/client/package.scala b/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/client/package.scala new file mode 100644 index 000000000..6d67fc515 --- /dev/null +++ b/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/client/package.scala @@ -0,0 +1,52 @@ +package org.elasticmq.rest.sqs + +package object client { + type QueueUrl = String + type Arn = String + type TaskHandle = String + type MessageMoveTaskStatus = String + type ApproximateNumberOfMessagesMoved = Long +} + +package client { + sealed abstract class QueueAttributeName(val value: String) + case object AllAttributeNames extends QueueAttributeName("All") + case object PolicyAttributeName extends QueueAttributeName("Policy") + case object VisibilityTimeoutAttributeName extends QueueAttributeName("VisibilityTimeout") + case object MaximumMessageSizeAttributeName extends QueueAttributeName("MaximumMessageSize") + case object MessageRetentionPeriodAttributeName extends QueueAttributeName("MessageRetentionPeriod") + case object ApproximateNumberOfMessagesAttributeName extends QueueAttributeName("ApproximateNumberOfMessages") + case object ApproximateNumberOfMessagesNotVisibleAttributeName + extends QueueAttributeName("ApproximateNumberOfMessagesNotVisible") + case object CreatedTimestampAttributeName extends QueueAttributeName("CreatedTimestamp") + case object LastModifiedTimestampAttributeName extends QueueAttributeName("LastModifiedTimestamp") + case object QueueArnAttributeName extends QueueAttributeName("QueueArn") + case object ApproximateNumberOfMessagesDelayedAttributeName + extends QueueAttributeName("ApproximateNumberOfMessagesDelayed") + case object DelaySecondsAttributeName extends QueueAttributeName("DelaySeconds") + case object ReceiveMessageWaitTimeSecondsAttributeName extends QueueAttributeName("ReceiveMessageWaitTimeSeconds") + case object RedrivePolicyAttributeName extends QueueAttributeName("RedrivePolicy") + case object FifoQueueAttributeName extends QueueAttributeName("FifoQueue") + case object ContentBasedDeduplicationAttributeName extends QueueAttributeName("ContentBasedDeduplication") + case object KmsMasterKeyIdAttributeName extends QueueAttributeName("KmsMasterKeyId") + case object KmsDataKeyReusePeriodSecondsAttributeName extends QueueAttributeName("KmsDataKeyReusePeriodSeconds") + case object DeduplicationScopeAttributeName extends QueueAttributeName("DeduplicationScope") + case object FifoThroughputLimitAttributeName extends QueueAttributeName("FifoThroughputLimit") + case object RedriveAllowPolicyAttributeName extends QueueAttributeName("RedriveAllowPolicy") + case object SqsManagedSseEnabledAttributeName extends QueueAttributeName("SqsManagedSseEnabled") + + case class ReceivedMessage(messageId: String, receiptHandle: String, body: String) + + case class MessageMoveTask( + taskHandle: TaskHandle, + sourceArn: Arn, + status: MessageMoveTaskStatus, + maxNumberOfMessagesPerSecond: Option[Int] + ) + + sealed trait SqsClientErrorType + case object UnsupportedOperation extends SqsClientErrorType + case object ResourceNotFound extends SqsClientErrorType + case object QueueDoesNotExist extends SqsClientErrorType + case object UnknownSqsClientErrorType extends SqsClientErrorType +} diff --git a/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/CancelMessageMoveTaskDirectives.scala b/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/CancelMessageMoveTaskDirectives.scala index 8aa78a2a8..9d16ab01b 100644 --- a/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/CancelMessageMoveTaskDirectives.scala +++ b/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/CancelMessageMoveTaskDirectives.scala @@ -6,6 +6,7 @@ import org.elasticmq.actor.reply._ import org.elasticmq.msg.CancelMessageMoveTask import org.elasticmq.rest.sqs.Action.{CancelMessageMoveTask => CancelMessageMoveTaskAction} import org.elasticmq.rest.sqs.Constants._ +import org.elasticmq.rest.sqs.SQSException.ElasticMQErrorOps import org.elasticmq.rest.sqs.directives.ElasticMQDirectives import org.elasticmq.rest.sqs.model.RequestPayload import spray.json.DefaultJsonProtocol._ @@ -22,7 +23,7 @@ trait CancelMessageMoveTaskDirectives { this: ElasticMQDirectives with QueueURLM await( queueManagerActor ? CancelMessageMoveTask(params.TaskHandle) ) match { - case Left(e: ElasticMQError) => throw new SQSException(e.code, errorMessage = Some(e.message)) + case Left(e: ElasticMQError) => throw e.toSQSException case Right(approximateNumberOfMessagesMoved) => complete(CancelMessageMoveTaskResponse(approximateNumberOfMessagesMoved)) } diff --git a/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/SQSException.scala b/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/SQSException.scala index 19edcfb89..7ca1c7f8a 100644 --- a/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/SQSException.scala +++ b/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/SQSException.scala @@ -1,5 +1,6 @@ package org.elasticmq.rest.sqs +import org.elasticmq.ElasticMQError import org.elasticmq.rest.sqs.Constants._ import scala.xml.Elem @@ -77,10 +78,14 @@ object SQSException { errorMessage = Some("The specified queue does not exist.") ) - def resourceNotFoundException: SQSException = - new SQSException( - "AWS.SimpleQueueService.ResourceNotFoundException", - errorType = "com.amazonaws.sqs#ResourceNotFoundException", - errorMessage = Some("One or more specified resources don't exist.") - ) + implicit class ElasticMQErrorOps(e: ElasticMQError) { + def toSQSException: SQSException = { + val errorType = e.code match { + case "AWS.SimpleQueueService.UnsupportedOperation" => Some("com.amazonaws.sqs#UnsupportedOperation") + case "ResourceNotFoundException" => Some("com.amazonaws.sqs#ResourceNotFoundException") + case _ => None + } + new SQSException(e.code, errorType = errorType.getOrElse("Sender"), errorMessage = Some(e.message)) + } + } } diff --git a/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/StartMessageMoveTaskDirectives.scala b/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/StartMessageMoveTaskDirectives.scala index 3125b7fa2..178fe9a23 100644 --- a/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/StartMessageMoveTaskDirectives.scala +++ b/rest/rest-sqs/src/main/scala/org/elasticmq/rest/sqs/StartMessageMoveTaskDirectives.scala @@ -7,6 +7,7 @@ import org.elasticmq.actor.reply._ import org.elasticmq.msg.StartMessageMoveTask import org.elasticmq.rest.sqs.Action.{StartMessageMoveTask => StartMessageMoveTaskAction} import org.elasticmq.rest.sqs.Constants._ +import org.elasticmq.rest.sqs.SQSException.ElasticMQErrorOps import org.elasticmq.rest.sqs.directives.ElasticMQDirectives import org.elasticmq.rest.sqs.model.RequestPayload import spray.json.DefaultJsonProtocol._ @@ -24,9 +25,16 @@ trait StartMessageMoveTaskDirectives extends ArnSupport { case Some(destinationQueueArn) => val destinationQueueName = extractQueueName(destinationQueueArn) queueActorAndDataFromQueueName(destinationQueueName) { (destinationQueue, _) => - startMessageMoveTask(sourceQueue, params.SourceArn, Some(destinationQueue), params.DestinationArn, params.MaxNumberOfMessagesPerSecond) + startMessageMoveTask( + sourceQueue, + params.SourceArn, + Some(destinationQueue), + params.DestinationArn, + params.MaxNumberOfMessagesPerSecond + ) } - case None => startMessageMoveTask(sourceQueue, params.SourceArn, None, None, params.MaxNumberOfMessagesPerSecond) + case None => + startMessageMoveTask(sourceQueue, params.SourceArn, None, None, params.MaxNumberOfMessagesPerSecond) } } } @@ -49,7 +57,7 @@ trait StartMessageMoveTaskDirectives extends ArnSupport { ) } yield { res match { - case Left(e: ElasticMQError) => throw new SQSException(e.code, errorMessage = Some(e.message)) + case Left(e: ElasticMQError) => throw e.toSQSException case Right(taskHandle) => complete(StartMessageMoveTaskResponse(taskHandle)) } } From fe2986842a9a69eaff69fa9ee24283de67f77354 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Ossowski?= Date: Mon, 8 Apr 2024 14:24:29 +0200 Subject: [PATCH 11/12] fix build --- .../scala/org/elasticmq/rest/sqs/client/AwsSdkV1SqsClient.scala | 2 +- .../scala/org/elasticmq/rest/sqs/client/AwsSdkV2SqsClient.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/client/AwsSdkV1SqsClient.scala b/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/client/AwsSdkV1SqsClient.scala index 5a81d2e4f..c896f5f69 100644 --- a/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/client/AwsSdkV1SqsClient.scala +++ b/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/client/AwsSdkV1SqsClient.scala @@ -3,7 +3,7 @@ package org.elasticmq.rest.sqs.client import com.amazonaws.services.sqs.AmazonSQS import com.amazonaws.services.sqs.model.{CancelMessageMoveTaskRequest, CreateQueueRequest, GetQueueAttributesRequest, ListMessageMoveTasksRequest, QueueDoesNotExistException, ReceiveMessageRequest, ResourceNotFoundException, SendMessageRequest, StartMessageMoveTaskRequest, UnsupportedOperationException} -import scala.jdk.CollectionConverters.{iterableAsScalaIterableConverter, mapAsJavaMapConverter, mapAsScalaMapConverter, seqAsJavaListConverter} +import scala.collection.JavaConverters._ class AwsSdkV1SqsClient(client: AmazonSQS) extends SqsClient { diff --git a/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/client/AwsSdkV2SqsClient.scala b/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/client/AwsSdkV2SqsClient.scala index 22d6cffb6..48fd325f9 100644 --- a/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/client/AwsSdkV2SqsClient.scala +++ b/rest/rest-sqs-testing-amazon-java-sdk/src/test/scala/org/elasticmq/rest/sqs/client/AwsSdkV2SqsClient.scala @@ -1,7 +1,7 @@ package org.elasticmq.rest.sqs.client import software.amazon.awssdk.services.sqs.model.{CancelMessageMoveTaskRequest, CreateQueueRequest, GetQueueAttributesRequest, ListMessageMoveTasksRequest, QueueDoesNotExistException, ReceiveMessageRequest, ResourceNotFoundException, SendMessageRequest, StartMessageMoveTaskRequest, UnsupportedOperationException, QueueAttributeName => AwsQueueAttributeName} -import scala.jdk.CollectionConverters.{iterableAsScalaIterableConverter, mapAsJavaMapConverter, mapAsScalaMapConverter, seqAsJavaListConverter} +import scala.collection.JavaConverters._ class AwsSdkV2SqsClient(client: software.amazon.awssdk.services.sqs.SqsClient) extends SqsClient { From c927aed69f9f3ad66324e4fa40d8d54cd37d4136 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Ossowski?= Date: Wed, 10 Apr 2024 16:18:31 +0200 Subject: [PATCH 12/12] refactoring --- .../scala/org/elasticmq/ElasticMQError.scala | 22 +++++-------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/core/src/main/scala/org/elasticmq/ElasticMQError.scala b/core/src/main/scala/org/elasticmq/ElasticMQError.scala index 855aa69c8..a8afe4dcd 100644 --- a/core/src/main/scala/org/elasticmq/ElasticMQError.scala +++ b/core/src/main/scala/org/elasticmq/ElasticMQError.scala @@ -1,45 +1,35 @@ package org.elasticmq import org.elasticmq.msg.MessageMoveTaskHandle -trait ElasticMQError { +sealed trait ElasticMQError { val queueName: String val code: String // TODO: code should be handled in rest-sqs module val message: String } -class QueueAlreadyExists(val queueName: String) extends ElasticMQError { +final case class QueueAlreadyExists(val queueName: String) extends ElasticMQError { val code = "QueueAlreadyExists" val message = s"Queue already exists: $queueName" } -case class QueueCreationError(queueName: String, reason: String) extends ElasticMQError { - val code = "QueueCreationError" - val message = s"Queue named $queueName could not be created because of $reason" -} - -case class InvalidParameterValue(queueName: String, reason: String) extends ElasticMQError { +final case class InvalidParameterValue(queueName: String, reason: String) extends ElasticMQError { val code = "InvalidParameterValue" val message = reason } -class MessageDoesNotExist(val queueName: String, messageId: MessageId) extends ElasticMQError { - val code = "MessageDoesNotExist" - val message = s"Message does not exist: $messageId in queue: $queueName" -} - -class InvalidReceiptHandle(val queueName: String, receiptHandle: String) extends ElasticMQError { +final case class InvalidReceiptHandle(val queueName: String, receiptHandle: String) extends ElasticMQError { val code = "ReceiptHandleIsInvalid" val message = s"""The receipt handle "$receiptHandle" is not valid.""" } -class InvalidMessageMoveTaskHandle(val taskHandle: MessageMoveTaskHandle) extends ElasticMQError { +final case class InvalidMessageMoveTaskHandle(val taskHandle: MessageMoveTaskHandle) extends ElasticMQError { val code = "ResourceNotFoundException" val message = s"""The task handle "$taskHandle" is not valid or does not exist""" override val queueName: String = "invalid" } -class MessageMoveTaskAlreadyRunning(val queueName: String) extends ElasticMQError { +final case class MessageMoveTaskAlreadyRunning(val queueName: String) extends ElasticMQError { val code = "AWS.SimpleQueueService.UnsupportedOperation" val message = s"""A message move task is already running on queue "$queueName"""" }