diff --git a/.travis.yml b/.travis.yml index 0b0a80e4..612cd0d9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,6 +15,22 @@ cache: directories: - $HOME/.gradle +before_script: + - mkdir 9000 9001 9002 9003 9004 9005 + - printf "port 9000 \ncluster-enabled yes \ncluster-config-file nodes.conf \ncluster-node-timeout 5000 \nappendonly yes" >> 9000/redis.conf + - printf "port 9001 \ncluster-enabled yes \ncluster-config-file nodes.conf \ncluster-node-timeout 5000 \nappendonly yes" >> 9001/redis.conf + - printf "port 9002 \ncluster-enabled yes \ncluster-config-file nodes.conf \ncluster-node-timeout 5000 \nappendonly yes" >> 9002/redis.conf + - printf "port 9003 \ncluster-enabled yes \ncluster-config-file nodes.conf \ncluster-node-timeout 5000 \nappendonly yes" >> 9003/redis.conf + - printf "port 9004 \ncluster-enabled yes \ncluster-config-file nodes.conf \ncluster-node-timeout 5000 \nappendonly yes" >> 9004/redis.conf + - printf "port 9005 \ncluster-enabled yes \ncluster-config-file nodes.conf \ncluster-node-timeout 5000 \nappendonly yes" >> 9005/redis.conf + - cd 9000 && redis-server ./redis.conf & + - cd 9001 && redis-server ./redis.conf & + - cd 9002 && redis-server ./redis.conf & + - cd 9003 && redis-server ./redis.conf & + - cd 9004 && redis-server ./redis.conf & + - cd 9005 && redis-server ./redis.conf & + - redis-cli --cluster create 127.0.0.1:9000 127.0.0.1:9001 127.0.0.1:9002 127.0.0.1:9003 127.0.0.1:9004 127.0.0.1:9005 --cluster-replicas 1 --cluster-yes + jobs: include: - stage: spring-boot-2.1 diff --git a/CHANGELOG.md b/CHANGELOG.md index b3939d71..b9fe4eb8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,20 +1,30 @@ -# Changelog -All notable changes to Rqueue project will be documented in this file. +# [Rqueue] New and Notable Changes -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), -and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +**NOTE**: The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [2.0.0] - TBD +## [2.0.1] - 17-May-2020 + +### Added +- Allow registering a queue, that can be in push only mode +- Apis to schedule task at the given time +- Refine enqueueIn apis to support Duration and TimeUnit + +### Fixes +- Arguments mismatch due to multiple class loaders. +- Dead letter queue clear lead to clearing all the messages related to that queue. + + +## [2.0.0] - 10-May-2020 ### Added - Web interface to visualize queue - Move message from one queue to another - Latency visualizer -- Delete message from the queue +- Delete one or more message(s) from the queue - Allow deactivating a consumer in a given environment - Single or multiple execution of polled messages - Queue level concurrency -- BackOff for failed message, linear or exponential +- BackOff for failed messages, linear or exponential - Group level queue priority - Multi level queue priority - Strict or weighted algorithm for message execution @@ -33,22 +43,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [1.3.2] - 01-Apr-2020 * Support lower version of spring 2.1.x - ## [1.3.1] - 27-Feb-2020 * **Fixed** Bootstrap issue due to optional dependencies of micrometer - ## [1.3] - 11-Dec-2019 -### Added -* Expose 6 queue metrics using micrometer. (queue-size, delay queue size, processing queue size, dead letter queue size, execution counter, failure counter) +* Expose multiple queue metrics using micrometer. (queue-size, delay queue size, processing queue size, dead letter queue size, execution counter, failure counter) * An api to move messages from dead letter queue to other queue. (Any source queue to target queue). ### Fixed -* An issue in scheduler that's always scheduling job at the delay of 5 seconds. (this leads to messages are not copied from delayed queue to main queue on high load) +* An issue in the scheduler that's always scheduling job at the delay of 5 seconds. (this leads to messages are not copied from delayed queue to main queue on high load) ## [1.2] - 03-Nov-2019 -* Fixed typo of *Later* to *Letter* +* Typo of *Later* to *Letter* ## [1.1] - 02-Nov-2019 @@ -57,7 +64,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Lua script to make atomic operation ## [1.0] - 23-Oct-2019 -* Basic version of Asynchronous task execution using Redis for Spring and Spring Boot +* The basic version of Asynchronous task execution using Redis for Spring and Spring Boot [1.0]: https://repo1.maven.org/maven2/com/github/sonus21/rqueue/1.0-RELEASE [1.1]: https://repo1.maven.org/maven2/com/github/sonus21/rqueue/1.1-RELEASE @@ -67,3 +74,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.3.2]: https://repo1.maven.org/maven2/com/github/sonus21/rqueue/1.3.2-RELEASE [1.4.0]: https://repo1.maven.org/maven2/com/github/sonus21/rqueue/1.4.0-RELEASE [2.0.0]: https://repo1.maven.org/maven2/com/github/sonus21/rqueue/2.0.0-RELEASE +[2.0.1]: https://repo1.maven.org/maven2/com/github/sonus21/rqueue/2.0.1-RELEASE diff --git a/README.md b/README.md index 82df5d56..15024cb4 100644 --- a/README.md +++ b/README.md @@ -5,45 +5,35 @@ [![Build Status](https://travis-ci.org/sonus21/rqueue.svg?branch=master)](https://travis-ci.org/sonus21/rqueue) [![Coverage Status](https://coveralls.io/repos/github/sonus21/rqueue/badge.svg?branch=master)](https://coveralls.io/github/sonus21/rqueue?branch=master) -![Maven Central](https://img.shields.io/maven-central/v/com.github.sonus21/rqueue) +[![Maven Central](https://img.shields.io/maven-central/v/com.github.sonus21/rqueue)](https://repo1.maven.org/maven2/com/github/sonus21/rqueue) [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](LICENSE) -Rqueue is an asynchronous task executor(worker) built for spring framework based on the spring framework's messaging library backed by Redis. +### Rqueue is an asynchronous task executor(worker) built for spring framework based on the spring framework's messaging library backed by Redis. -### Some of the features +## Features -* A message can be delayed for an arbitrary period of time or delivered immediately. +* A message can be delayed for an arbitrary period or delivered immediately. * Multiple messages can be consumed in parallel by different workers. * Message delivery: It's guaranteed that a message is consumed **at least once**. (Message would be consumed by a worker more than once due to the failure in the underlying worker/restart-process etc, otherwise exactly one delivery) * Support Redis cluster * Queue metrics * Different Redis connection for application and worker -* Web interface for queue management and queue stats +* Web interface for queue management and queue statistics * Automatic message serialization and deserialization -* Consumer concurrency -* Group level queue priority( weighted and strict) -* Sub queue priority (weighted and strict) -* Task execution back off, exponential and fixed back off +* Queue concurrency +* Group level queue priority(weighted and strict) +* Sub queue priority(weighted and strict) +* Task execution back off, exponential and fixed back off (default fixed back off) +* Callbacks for different actions +* Events 1. Bootstrap event 2. Task execution event. +#### NOTE: For Redis cluster, it's required to provide MASTER [connection](https://github.com/sonus21/rqueue/issues/19) -### Adding a task -Rqueue supports two types of tasks. -1. Execute tasks as soon as possible -2. Delayed tasks (task that would be scheduled at given delay, run task at 2:30PM) - -### Task execution configuration -Task execution can be configured in different ways -1. By default a tasks would be retried for Integer.MAX_VALUE number of times -2. If we do not need retry then we need to set retry count to zero -3. After retrying/executing a task N (>=1) times if we can't execute the given task then the task can be discarded or push to dead-letter-queue - -## Usage - -### Configuration +## Configuration -#### Spring-boot +### Spring-boot -* Get latest one from [Maven central](https://search.maven.org/search?q=g:com.github.sonus21%20AND%20a:rqueue-spring-boot-starter) +* Get the latest one from [Maven central](https://search.maven.org/search?q=g:com.github.sonus21%20AND%20a:rqueue-spring-boot-starter) * Add dependency * Gradle ```groovy @@ -58,9 +48,9 @@ Task execution can be configured in different ways ``` -#### Spring framework +### Spring framework -* Get latest one from [Maven central](https://search.maven.org/search?q=g:com.github.sonus21%20AND%20a:rqueue-spring) +* Get the latest one from [Maven central](https://search.maven.org/search?q=g:com.github.sonus21%20AND%20a:rqueue-spring) * Add Dependency * Gradle ```groovy @@ -89,10 +79,60 @@ public class Application{ ``` +## Message publishing/Task submission +Rqueue supports two types of tasks. +1. Execute tasks as soon as possible +2. Delayed tasks (task that would be scheduled at given time or run in 5 minutes) + +All messages need to be sent using `RqueueMessageSender` bean's `enqueueXXX`, `enqueueInXXX` and `enqueueAtXXX` methods. +It has handful number of `enqueue`, `enqueueIn`, `enqueueAt` methods, we can use one of them based on the use case. -### Worker/Task executor +```java +public class MessageService { + @AutoWired private RqueueMessageSender rqueueMessageSender; + + public void doSomething(){ + rqueueMessageSender.enqueue("simple-queue", "Rqueue is configured"); + } + + public void createJOB(Job job){ + rqueueMessageSender.enqueue("job-queue", job); + } + + // send notification in 30 seconds + public void sendNotification(Notification notification){ + rqueueMessageSender.enqueueIn("notification-queue", notification, 30*1000L); + } + + // enqueue At example + public void createInvoice(Invoice invoice, Instant instant){ + rqueueMessageSender.enqueueAt("invoice-queue", invoice, instant); + } + + // enqueue with priority, when sub queues are used as explained in the queue priority section. + enum SmsPriority{ + CRITICAL("critical"), + HIGH("high"), + MEDIUM("medium"), + LOW("low"); + private String value; + } + public void sendSms(Sms sms, SmsPriority priority){ + rqueueMessageSender.enqueueWithPriority("sms-queue", priority.value(), sms); + } +} +``` + + +## Worker/Consumer/Task executor/Listener Any method that's part of spring bean, can be marked as worker/message listener using `RqueueListener` annotation +**Task execution policy:** + +1. By default, all tasks would be retried for Integer.MAX_VALUE number of times, queue having dead letter queue set would be retried for 3 times, if retry count is not provided. +2. If we do not need any retry then we need to set retry count to zero +3. After retrying/executing a task N (>=1) times if we can't execute the given task then the task can be discarded or push to dead-letter-queue + ```java @Component @@ -103,62 +143,87 @@ public class MessageListener { log.info("simple-queue: {}", message); } - // Scheduled Job notification - @RqueueListener(value = "job-queue", - numRetries="3", deadLetterQueue="failed-job-queue") + @RqueueListener(value = "job-queue", numRetries="3", + deadLetterQueue="failed-job-queue", concurrency="5-10") public void onMessage(Job job) { - log.info("Job created: {}", job); + log.info("Job alert: {}", job); } - // Scheduled push notification - @RqueueListener(value = "push-notification-queue", - numRetries="3", deadLetterQueue="failed-notification-queue") + @RqueueListener(value = "push-notification-queue",numRetries="3", + deadLetterQueue="failed-notification-queue") public void onMessage(Notification notification) { - log.info("Notification message: {}", notification); + log.info("Push notification: {}", notification); } - // asynchronously send sms to the user @RqueueListener(value = "sms", priority="critical=10,high=8,medium=4,low=1") public void onMessage(Sms sms) { - log.info("Sms message: {}", sms); + log.info("Sms : {}", sms); } -} -``` - -### Message publishing or task submission -All messages can be send using `RqueueMessageSender` bean's methods. It has handful number of `enqueue` and `enqueueIn` methods, we can use one of them based on the use case. - - -```java -public class MessageService { - @AutoWired private RqueueMessageSender rqueueMessageSender; - public void doSomething(){ - rqueueMessageSender.enqueue("simple-queue", "Rqueue is configured"); + @RqueueListener(value = "chat-indexing", priority="20", priorityGroup="chat") + public void onMessage(ChatIndexing chatIndexing) { + log.info("ChatIndexing message: {}", chatIndexing); } - public void createJOB(Job job){ - //do something - rqueueMessageSender.enqueue("job-queue", job); + @RqueueListener(value = "chat-indexing-daily", priority="10", priorityGroup="chat") + public void onMessage(ChatIndexing chatIndexing) { + log.info("ChatIndexing message: {}", chatIndexing); } - // send notification in 30 seconds - public void sendNotification(Notification notification){ - //do something - rqueueMessageSender.enqueueIn("notification-queue", notification, 30*1000L); + @RqueueListener(value = "chat-indexing-weekly", priority="5", priorityGroup="chat") + public void onMessage(ChatIndexing chatIndexing) { + log.info("ChatIndexing message: {}", chatIndexing); } } ``` -#### ** Key points: ** +** Key points: ** * A task would be retried without any further configuration * Method arguments are handled automatically, as in above example even task of `Job` type can be executed by workers. +## Queue Priority +Rqueue supports two types of priority + +1. Weighted +2. Strict + +You need to set `priorityMode` in container factory either `STRICT` or `WEIGHTED`. + + +Priority needs to be specified using annotation's field `priority`, priority can be grouped for multiple queues as well, +by default any queue having priority is added to the default priority group. +Queue can be classified into multiple priority groups as well for example sms queue has 4 types of sub queues +`critical`, `high`, `medium` and `low` their priority needs to be specified as `priority="critical=10, high=6, medium=4,low=1"`. + +If you want to use simple priority mechanism, then you need to set priority to some number and their group, for example. + +`priority="60"` +`priorityGroup="critical-group"` + +Let's say there're three queues + +|Queue|Priority| +|-----|--------| +|Q1 | 6 | +|Q2 | 4 | +|Q3 | 2 | + +#### Weighted Priority +Weighted priority works in round robin fashion, in this example Q1 would be polled 6 times, Q2 4 times and Q3 2 times and again this process repeats. After polling if it's found that any queue does not have more jobs than their weight is reduced by Δ e.g Q1 does not have any item then it's weight is reduces by `Δ = currentWeight * (1-(6/(6+4+2))`. As soon as the weight becomes <= 0, queue is inactive and weight is reinitialised when all queues become inactive. + +This algorithm is implemented in [WeightedPriorityPoller](https://github.com/sonus21/rqueue/blob/652ec46c52dc6f06604d500c15ac11a8633eb602/rqueue/src/main/java/com/github/sonus21/rqueue/listener/WeightedPriorityPoller.java) + +#### Strict Priority +In Strict priority case, poller would always first poll from the highest priority queue, that's Q1 here. After polling if it encounters that queue does not have any element than that queue becomes inactive for **polling interval**, to avoid starvation a queue can be inactive for maximum of 1 minute. + + +This algorithm is implemented in [StrictPriorityPoller](https://github.com/sonus21/rqueue/blob/652ec46c52dc6f06604d500c15ac11a8633eb602/rqueue/src/main/java/com/github/sonus21/rqueue/listener/StrictPriorityPoller.java) + ## Monitoring Queue Statistics NOTE: Rqueue support micrometer library for monitoring. -**It provides 4 types gauge of metrics.** +**It provides 4 types of gauge metrics.** * **queue.size** : number of tasks to be run * **dead.letter.queue.size** : number of tasks in the dead letter queue @@ -177,14 +242,14 @@ We need to set count.execution and count.failure fields of RqueueMetricsProperti All these metrics are tagged **Spring Boot Application** -1. Add micrometer and the exporter dependencies +1. Add `micrometer` and the exporter dependencies 2. Set tags if any using `rqueue.metrics.tags. = ` 3. Enable counting features using `rqueue.metrics.count.execution=true`, `rqueue.metrics.count.failure=true` **Spring Application** -1. Add micrometer and the exporter dependencies provide MeterRegistry as bean -2. Provide bean of RqueueMetricsProperties, in this bean set all the required fields. +1. Add `micrometer` and the exporter dependencies provide `MeterRegistry` as bean +2. Provide a bean of `RqueueMetricsProperties`, in this bean set all the required fields. [![Grafana Dashboard](https://raw.githubusercontent.com/sonus21/rqueue/master/docs/static/grafana-dashboard.png)](https://raw.githubusercontent.com/sonus21/rqueue/master/docs/static/grafana-dashboard.png) @@ -192,7 +257,7 @@ All these metrics are tagged ## Web Interface * Queue stats (Scheduled, waiting to run, running, moved to dead letter queue) -* Latency: Min/Max and Average task execution time +* Latency: Min/Max and Average task execution time * Queue Management: Move tasks from one queue to another * Delete enqueued tasks * Enqueued tasks insights @@ -224,19 +289,19 @@ Link: [http://localhost:8080/rqueue](http://localhost:8080/rqueue) All paths are under `/rqueue/**`. for authentication add interceptor(s) that would check for the session etc. -## Demo Applications +## Example Applications 1. [Rqueue Spring Boot-1](https://github.com/sonus21/rqueue/tree/master/rqueue-spring-boot-example) -2. [Rqueue Spring Boot-2](https://github.com/sonus21/rqueue-task-exector) -2. [Rqueue Spring ](https://github.com/sonus21/rqueue/tree/master/rqueue-spring-example) +2. [Rqueue Spring Boot-2](https://github.com/sonus21/rqueue-task-executor) +3. [Rqueue Spring](https://github.com/sonus21/rqueue/tree/master/rqueue-spring-example) ## Advance Configuration -Apart from basic configuration, it can be customized heavily, like number of tasks it would be executing concurrently. +Apart from the basic configuration, it can be customized heavily, like number of tasks it would be executing concurrently. More and more configurations can be provided using `SimpleRqueueListenerContainerFactory` class. ```java -class Application{ +class RqueueConfiguration{ @Bean public SimpleRqueueListenerContainerFactory simpleRqueueListenerContainerFactory(){ // return SimpleRqueueListenerContainerFactory object @@ -245,69 +310,112 @@ class Application{ ``` --- -**Configure Task store** - All tasks are stored in Redis database, either we can utilise the same Redis database for entire application or we can provide a separate one for task store. Nonetheless if we need different database then we can configure using `setRedisConnectionFactory` method. - **It's highly recommended to provide master connection as it reads and writes to the Redis database** + +### Task store/Redis Connection + +All tasks are stored in Redis database. As an application developer, you have the flexibility to specify the redis connection to be used by Rqueue. +You need to specify redis connection via container factory's method `setRedisConnectionFactory`. ```java +class RqueueConfiguration { @Bean - public SimpleRqueueListenerContainerFactory simpleRqueueListenerContainerFactory(RqueueMessageHandler rqueueMessageHandler){ - // Create redis connection factory of your choice either Redission or Lettuce or Jedis - // Get redis configuration - RedisConfiguration redisConfiguration = new RedisConfiguration(); - // Set fields of redis configuration + public SimpleRqueueListenerContainerFactory simpleRqueueListenerContainerFactory(){ + SimpleRqueueListenerContainerFactory factory = new SimpleRqueueListenerContainerFactory(); + // StandAlone redis can use Jedis connection factory but for clustered redis, it's required to use Lettuce + // because EVALSHA will not work with Jedis. + + // Stand alone redis configuration, Set fields of redis configuration + RedisStandaloneConfiguration redisConfiguration = new RedisStandaloneConfiguration(); + + // Redis cluster, set required fields + RedisClusterConfiguration redisConfiguration = new RedisClusterConfiguration(); + // Create lettuce connection factory - LettuceConnectionFactory lettuceConnectionFactory = new LettuceConnectionFactory(redisConfiguration); - factory.setRedisConnectionFactory(lettuceConnectionFactory); - facory.setRqueueMessageHandler(rqueueMessageHandler); + LettuceConnectionFactory redisConnectionFactory = new LettuceConnectionFactory(redisConfiguration); + redisConnectionFactory.afterPropertiesset(); + factory.setRedisConnectionFactory(redisConnectionFactory); return factory; } +} ``` ---- -**Redis connection failure and retry** +--- +### Redis connection failure and retry Whenever a call to Redis failed then it would be retried in 5 seconds, to change that we can set back off to some different value. ```java -// set backoff time to 100 milli second -factory.setBackOffTime( 100 ); +class RqueueConfiguration { + @Bean + public SimpleRqueueListenerContainerFactory simpleRqueueListenerContainerFactory(){ + SimpleRqueueListenerContainerFactory factory = new SimpleRqueueListenerContainerFactory(); + //... + // set back off time to 100 milli second + factory.setBackOffTime( 100 ); + return factory; + } +} ``` ---- +### Task executor/Queue Concurrency +The number of task executors are twice of the number of queues. +A custom or shared task executor can be configured using factory's `setTaskExecutor` method, +we need to provide an implementation of `AsyncTaskExecutor` class. +It's possible to provide queue concurrency using `RqueueListener` annotation's field `concurrency`. +The concurrency could be some positive number like 10, it could be in range as well 5-10. +If queue concurrency is provided then each queue will use their own task executor to execute consumed tasks, otherwise a shared task executor is used to execute tasks. -**Task executor** - -Number of workers can be configured using `setMaxNumWorkers` method. For example to configure 10 workers we can do +A global number of workers can be configured using `setMaxNumWorkers` method. + ```java -SimpleRqueueListenerContainerFactory factory = new SimpleRqueueListenerContainerFactory(); -factory.setMaxNumWorkers(10); +class RqueueConfiguration { + @Bean + public SimpleRqueueListenerContainerFactory simpleRqueueListenerContainerFactory(){ + SimpleRqueueListenerContainerFactory factory = new SimpleRqueueListenerContainerFactory(); + //... + factory.setMaxNumWorkers(10); + return factory; + } +} ``` -By default number of task executors are twice of the number of queues. A custom or shared task executor can be configured using factory's `setTaskExecutor` method, we need to provide an implementation of AsyncTaskExecutor - - ```java -ThreadPoolTaskExecutor threadPoolTaskExecutor = new ThreadPoolTaskExecutor(); -threadPoolTaskExecutor.setThreadNamePrefix( "taskExecutor" ); -threadPoolTaskExecutor.setCorePoolSize(10); -threadPoolTaskExecutor.setMaxPoolSize(50); -threadPoolTaskExecutor.setQueueCapacity(0); -threadPoolTaskExecutor.afterPropertiesSet(); -factory.setTaskExecutor(threadPoolTaskExecutor); +class RqueueConfiguration { + @Bean + public SimpleRqueueListenerContainerFactory simpleRqueueListenerContainerFactory(){ + SimpleRqueueListenerContainerFactory factory = new SimpleRqueueListenerContainerFactory(); + //... + ThreadPoolTaskExecutor threadPoolTaskExecutor = new ThreadPoolTaskExecutor(); + threadPoolTaskExecutor.setThreadNamePrefix( "taskExecutor" ); + threadPoolTaskExecutor.setCorePoolSize(10); + threadPoolTaskExecutor.setMaxPoolSize(50); + threadPoolTaskExecutor.setQueueCapacity(0); + threadPoolTaskExecutor.afterPropertiesSet(); + factory.setTaskExecutor(threadPoolTaskExecutor); + return factory; + } +} ``` --- -**Manual/Auto start of the container** -Whenever container is refreshed or application is started then it is be started automatically and it comes with a graceful shutdown. -Automatic start of the container can be controlled using `autoStartup` flag, when autoStartup is false then application must call start and stop -method of container to start and stop the container, for further graceful shutdown you should call destroy method as well. -```java -factory.setAutoStartup(false); +### Manual/Auto start of the container +Whenever container is refreshed or application is started then it is started automatically, it also comes with a graceful shutdown. +Automatic start of the container can be controlled using `autoStartup` flag, when autoStartup is false then application must call start and stop methods of container. +For further graceful shutdown application should call destroy method as well. +```java +class RqueueConfiguration { + @Bean + public SimpleRqueueListenerContainerFactory simpleRqueueListenerContainerFactory(){ + SimpleRqueueListenerContainerFactory factory = new SimpleRqueueListenerContainerFactory(); + //... + factory.setAutoStartup(false); + return factory; + } +} ``` ```java @@ -332,29 +440,42 @@ public class BootstrapController { --- -**Message converters configuration** +### Message converters configuration Generally any message can be converted to and from without any problems, though it can be customized by providing an implementation `org.springframework.messaging.converter.MessageConverter`, this message converter must implement both the methods of `MessageConverter` interface. Implementation must make sure the return type of method `toMessage` is `Message` while as in the case of `fromMessage` a object can be returned as well. ```java -MessageConverter messageConverter = new SomeFancyMessageConverter(); -List messageConverters = new ArrayList<>(); -messageConverters.add(messageConverter); -factory.setMessageConverters(messageConverters); +class RqueueConfiguration { + @Bean + public SimpleRqueueListenerContainerFactory simpleRqueueListenerContainerFactory(){ + SimpleRqueueListenerContainerFactory factory = new SimpleRqueueListenerContainerFactory(); + //... + MessageConverter messageConverter = new SomeFancyMessageConverter(); + List messageConverters = new ArrayList<>(); + messageConverters.add(messageConverter); + factory.setMessageConverters(messageConverters); + return factory; + } +} ``` More than one message converter can be used as well, when more than one message converters are provided then they are used in the order, whichever returns **non null** value is used. -## Migration from 1.x to 2.0 -**Set redis key __rq::version=1**,this is required to handles data present in the old queues. Rqueue would attempt to identify version from the Redis keys, if it can't determine then it will use 2 as the default version. +## Migration from 1.x to 2.x +Set redis key **__rq::version=1** or add `rqueue.db.version=1` in the properties file, this is required to handle data present in the old queues. +It's safe to use version 2.x if existing queues do not have any tasks to process. Look for the results of following redis commands. +``` +1. LLEN +2. ZCARD rqueue-delay:: +3. ZCARD rqueue-processing:: +``` +If all of these commands gives zero then all tasks have been consumed. ## Support -Please report problem, bug or feature(s) to [issue](https://github.com/sonus21/rqueue/issues/new/choose) tracker. You are most welcome for any pull requests for feature, issue or enhancement. +Please report bug,question,feature(s) to [issue](https://github.com/sonus21/rqueue/issues/new/choose) tracker. You are most welcome for any pull requests for any feature/bug/enhancement. ## License © [Sonu Kumar](mailto:sonunitw12@gmail.com) 2019-Instant.now The Rqueue is released under version 2.0 of the Apache License. - - diff --git a/build.gradle b/build.gradle index 3fce7aed..73dbc725 100644 --- a/build.gradle +++ b/build.gradle @@ -67,7 +67,7 @@ ext { subprojects { group = 'com.github.sonus21' - version = '2.0.0-RELEASE' + version = '2.0.1-RELEASE' dependencies { // https://mvnrepository.com/artifact/org.springframework/spring-messaging diff --git a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/MessageListener.java b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/MessageListener.java index 046869a1..8a16b3a3 100644 --- a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/MessageListener.java +++ b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/MessageListener.java @@ -17,19 +17,23 @@ package com.github.sonus21.rqueue.test; import com.github.sonus21.rqueue.annotation.RqueueListener; +import com.github.sonus21.rqueue.core.RqueueMessage; +import com.github.sonus21.rqueue.listener.RqueueMessageHeaders; import com.github.sonus21.rqueue.test.dto.ChatIndexing; import com.github.sonus21.rqueue.test.dto.Email; import com.github.sonus21.rqueue.test.dto.FeedGeneration; import com.github.sonus21.rqueue.test.dto.Job; import com.github.sonus21.rqueue.test.dto.Notification; -import com.github.sonus21.rqueue.test.dto.Sms; import com.github.sonus21.rqueue.test.dto.Reservation; +import com.github.sonus21.rqueue.test.dto.Sms; import com.github.sonus21.rqueue.test.service.ConsumedMessageService; import com.github.sonus21.rqueue.test.service.FailureManager; import lombok.NonNull; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.messaging.handler.annotation.Header; +import org.springframework.messaging.handler.annotation.Payload; import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Transactional; @@ -54,8 +58,10 @@ public void onMessage(Job job) throws Exception { value = "${notification.queue.name}", numRetries = "${notification.queue.retry.count}", active = "${notification.queue.active}") - public void onMessage(Notification notification) throws Exception { - log.info("Notification: {}", notification); + public void onMessage( + @Payload Notification notification, @Header(RqueueMessageHeaders.ID) String id) + throws Exception { + log.info("Notification: {}, Id: {}", notification, id); if (failureManager.shouldFail(notification.getId())) { throw new Exception("Failing notification task to be retried" + notification); } @@ -68,8 +74,9 @@ public void onMessage(Notification notification) throws Exception { numRetries = "${email.queue.retry.count}", visibilityTimeout = "${email.execution.time}", active = "${email.queue.active}") - public void onMessage(Email email) throws Exception { - log.info("Email: {}", email); + public void onMessage(Email email, @Header(RqueueMessageHeaders.MESSAGE) RqueueMessage message) + throws Exception { + log.info("Email: {} Message: {}", email, message); if (failureManager.shouldFail(email.getId())) { throw new Exception("Failing email task to be retried" + email); } diff --git a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/BaseApplication.java b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/application/BaseApplication.java similarity index 98% rename from rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/BaseApplication.java rename to rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/application/BaseApplication.java index 03ccebe8..392e550f 100644 --- a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/BaseApplication.java +++ b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/application/BaseApplication.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.github.sonus21.rqueue.test; +package com.github.sonus21.rqueue.test.application; import com.fasterxml.jackson.databind.ObjectMapper; import javax.annotation.PostConstruct; diff --git a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/application/MultiRedisSprigBaseApplication.java b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/application/MultiRedisSprigBaseApplication.java new file mode 100644 index 00000000..41a7da2c --- /dev/null +++ b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/application/MultiRedisSprigBaseApplication.java @@ -0,0 +1,160 @@ +/* + * Copyright 2020 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.sonus21.rqueue.test.application; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.sonus21.rqueue.config.SimpleRqueueListenerContainerFactory; +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import javax.sql.DataSource; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.data.redis.connection.RedisStandaloneConfiguration; +import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory; +import org.springframework.data.redis.listener.RedisMessageListenerContainer; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; +import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean; +import org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter; +import redis.embedded.RedisServer; + +@Slf4j +public abstract class MultiRedisSprigBaseApplication { + + @Value("${spring.redis2.port}") + private int redisPort2; + + @Value("${spring.redis2.host}") + private String redisHost2; + + private RedisServer redisServer; + + @Value("${mysql.db.name}") + private String dbName; + + @Value("${spring.redis.port}") + private int redisPort; + + @Value("${spring.redis.host}") + private String redisHost; + + @Value("${use.system.redis:false}") + private boolean useSystemRedis; + + private RedisServer redisServer2; + + private ExecutorService executor; + private List lines = new ArrayList<>(); + Process process; + + @PostConstruct + public void postConstruct() { + if (redisServer == null) { + redisServer = new RedisServer(redisPort); + redisServer.start(); + } + if (redisServer2 == null) { + redisServer2 = new RedisServer(redisPort2); + redisServer2.start(); + } + executor = Executors.newSingleThreadExecutor(); + executor.submit( + () -> { + try { + process = Runtime.getRuntime().exec("redis-cli -p " + redisPort + " monitor"); + BufferedReader br = new BufferedReader(new InputStreamReader(process.getInputStream())); + String s; + while ((s = br.readLine()) != null) { + lines.add(s); + } + process.waitFor(); + } catch (Exception e) { + log.error("Process call failed", e); + } + }); + } + + @PreDestroy + public void preDestroy() { + if (redisServer != null) { + redisServer.stop(); + } + if (redisServer2 != null) { + redisServer2.stop(); + } + if (process != null) { + process.destroy(); + } + for (String line : lines) { + assert line.equals("OK"); + } + } + + @Bean + public LettuceConnectionFactory redisConnectionFactory() { + return new LettuceConnectionFactory(redisHost, redisPort2); + } + + @Bean + public DataSource dataSource() { + EmbeddedDatabaseBuilder builder = new EmbeddedDatabaseBuilder(); + return builder.setType(EmbeddedDatabaseType.H2).setName(dbName).build(); + } + + @Bean + public LocalContainerEntityManagerFactoryBean entityManagerFactory() { + HibernateJpaVendorAdapter vendorAdapter = new HibernateJpaVendorAdapter(); + vendorAdapter.setGenerateDdl(true); + LocalContainerEntityManagerFactoryBean factory = new LocalContainerEntityManagerFactoryBean(); + factory.setJpaVendorAdapter(vendorAdapter); + factory.setPackagesToScan("com.github.sonus21.rqueue.test.entity"); + factory.setDataSource(dataSource()); + return factory; + } + + @Bean + public RedisMessageListenerContainer container(RedisConnectionFactory redisConnectionFactory) { + RedisMessageListenerContainer container = new RedisMessageListenerContainer(); + container.setConnectionFactory(redisConnectionFactory); + return container; + } + + @Bean + public ObjectMapper objectMapper() { + return new ObjectMapper(); + } + + @Bean + public SimpleRqueueListenerContainerFactory simpleRqueueListenerContainerFactory() { + SimpleRqueueListenerContainerFactory simpleRqueueListenerContainerFactory = + new SimpleRqueueListenerContainerFactory(); + RedisStandaloneConfiguration redisConfiguration = + new RedisStandaloneConfiguration(redisHost2, redisPort2); + LettuceConnectionFactory connectionFactory = new LettuceConnectionFactory(redisConfiguration); + connectionFactory.afterPropertiesSet(); + simpleRqueueListenerContainerFactory.setRedisConnectionFactory(connectionFactory); + return simpleRqueueListenerContainerFactory; + } +} diff --git a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/application/RedisClusterBaseApplication.java b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/application/RedisClusterBaseApplication.java new file mode 100644 index 00000000..c71339b7 --- /dev/null +++ b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/application/RedisClusterBaseApplication.java @@ -0,0 +1,84 @@ +/* + * Copyright 2020 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.sonus21.rqueue.test.application; + +import com.github.sonus21.rqueue.config.SimpleRqueueListenerContainerFactory; +import com.github.sonus21.rqueue.listener.RqueueMessageHandler; +import com.github.sonus21.rqueue.utils.Constants; +import io.lettuce.core.ReadFrom; +import java.util.ArrayList; +import java.util.List; +import javax.sql.DataSource; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.data.redis.connection.RedisClusterConfiguration; +import org.springframework.data.redis.connection.RedisNode; +import org.springframework.data.redis.connection.RedisServer; +import org.springframework.data.redis.connection.lettuce.LettuceClientConfiguration; +import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; +import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean; +import org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter; + +public abstract class RedisClusterBaseApplication { + @Value("${mysql.db.name}") + private String dbName; + + @Bean + public DataSource dataSource() { + EmbeddedDatabaseBuilder builder = new EmbeddedDatabaseBuilder(); + return builder.setType(EmbeddedDatabaseType.H2).setName(dbName).build(); + } + + @Bean + public LocalContainerEntityManagerFactoryBean entityManagerFactory() { + HibernateJpaVendorAdapter vendorAdapter = new HibernateJpaVendorAdapter(); + vendorAdapter.setGenerateDdl(true); + LocalContainerEntityManagerFactoryBean factory = new LocalContainerEntityManagerFactoryBean(); + factory.setJpaVendorAdapter(vendorAdapter); + factory.setPackagesToScan("com.github.sonus21.rqueue.test.entity"); + factory.setDataSource(dataSource()); + return factory; + } + + @Bean + public SimpleRqueueListenerContainerFactory simpleRqueueListenerContainerFactory( + RqueueMessageHandler rqueueMessageHandler) { + LettuceClientConfiguration lettuceClientConfiguration = + LettuceClientConfiguration.builder().readFrom(ReadFrom.MASTER).build(); + + RedisClusterConfiguration redisClusterConfiguration = new RedisClusterConfiguration(); + List redisNodes = new ArrayList<>(); + redisNodes.add(new RedisNode("127.0.0.1", 9000)); + redisNodes.add(new RedisNode("127.0.0.1", 9001)); + redisNodes.add(new RedisNode("127.0.0.1", 9002)); + redisNodes.add(new RedisNode("127.0.0.1", 9003)); + redisNodes.add(new RedisNode("127.0.0.1", 9004)); + redisNodes.add(new RedisNode("127.0.0.1", 9005)); + redisClusterConfiguration.setClusterNodes(redisNodes); + LettuceConnectionFactory lettuceConnectionFactory = + new LettuceConnectionFactory(redisClusterConfiguration, lettuceClientConfiguration); + lettuceConnectionFactory.afterPropertiesSet(); + SimpleRqueueListenerContainerFactory simpleRqueueListenerContainerFactory = + new SimpleRqueueListenerContainerFactory(); + simpleRqueueListenerContainerFactory.setRedisConnectionFactory(lettuceConnectionFactory); + simpleRqueueListenerContainerFactory.setPollingInterval(Constants.ONE_MILLI); + simpleRqueueListenerContainerFactory.setRqueueMessageHandler(rqueueMessageHandler); + return simpleRqueueListenerContainerFactory; + } +} diff --git a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/SpringTestBase.java b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/common/SpringTestBase.java similarity index 90% rename from rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/SpringTestBase.java rename to rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/common/SpringTestBase.java index ebcbec33..3dc03969 100644 --- a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/SpringTestBase.java +++ b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/common/SpringTestBase.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.github.sonus21.rqueue.test.tests; +package com.github.sonus21.rqueue.test.common; import com.github.sonus21.rqueue.common.RqueueRedisTemplate; import com.github.sonus21.rqueue.config.RqueueConfig; @@ -28,6 +28,7 @@ import com.github.sonus21.rqueue.listener.RqueueMessageListenerContainer; import com.github.sonus21.rqueue.test.service.ConsumedMessageService; import com.github.sonus21.rqueue.test.service.FailureManager; +import com.github.sonus21.rqueue.utils.StringUtils; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -38,7 +39,7 @@ import org.springframework.beans.factory.annotation.Value; @Slf4j -public abstract class SpringTestBase { +public abstract class SpringTestBase extends TestBase { @Autowired protected RqueueMessageSender rqueueMessageSender; @Autowired protected RqueueMessageTemplate rqueueMessageTemplate; @Autowired protected RqueueConfig rqueueConfig; @@ -84,14 +85,6 @@ protected void enqueue(Object message, String queueName) { rqueueMessageTemplate.addMessage(queueName, rqueueMessage); } - public interface Factory { - Object next(int i); - } - - public interface Delay { - long getDelay(int i); - } - protected void enqueue(String queueName, Factory factory, int n) { for (int i = 0; i < n; i++) { Object object = factory.next(i); @@ -156,4 +149,22 @@ protected void printQueueStats(List queueNames) { } } } + + protected void cleanQueue(String queue) { + QueueDetail queueDetail = QueueRegistry.get(queue); + stringRqueueRedisTemplate.delete(queueDetail.getQueueName()); + stringRqueueRedisTemplate.delete(queueDetail.getDelayedQueueName()); + stringRqueueRedisTemplate.delete(queueDetail.getProcessingQueueName()); + if (!StringUtils.isEmpty(queueDetail.getDeadLetterQueueName())) { + stringRqueueRedisTemplate.delete(queueDetail.getDeadLetterQueueName()); + } + } + + public interface Factory { + Object next(int i); + } + + public interface Delay { + long getDelay(int i); + } } diff --git a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/SpringWebTestBase.java b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/common/SpringWebTestBase.java similarity index 88% rename from rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/SpringWebTestBase.java rename to rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/common/SpringWebTestBase.java index a24e4be5..e858b443 100644 --- a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/SpringWebTestBase.java +++ b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/common/SpringWebTestBase.java @@ -14,10 +14,9 @@ * limitations under the License. */ -package com.github.sonus21.rqueue.test.tests; +package com.github.sonus21.rqueue.test.common; import com.fasterxml.jackson.databind.ObjectMapper; -import com.github.sonus21.rqueue.exception.TimedOutException; import org.junit.Before; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.web.servlet.MockMvc; @@ -30,7 +29,7 @@ public abstract class SpringWebTestBase extends SpringTestBase { protected ObjectMapper mapper = new ObjectMapper(); @Before - public void init() throws TimedOutException { + public void init() { this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).build(); } } diff --git a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/common/TestBase.java b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/common/TestBase.java new file mode 100644 index 00000000..63062661 --- /dev/null +++ b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/common/TestBase.java @@ -0,0 +1,37 @@ +/* + * Copyright 2020 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.sonus21.rqueue.test.common; + +import java.util.Random; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public abstract class TestBase { + protected static Random random; + + static { + String seed = System.getenv("TEST_SEED"); + long randomSeed; + if (seed == null) { + randomSeed = System.currentTimeMillis(); + } else { + randomSeed = Long.parseLong(seed); + } + random = new Random(randomSeed); + log.info("Test random seed is {}", randomSeed); + } +} diff --git a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/AllQueueMode.java b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/AllQueueMode.java index d7aa8538..ea362da1 100644 --- a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/AllQueueMode.java +++ b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/AllQueueMode.java @@ -17,15 +17,17 @@ package com.github.sonus21.rqueue.test.tests; import com.github.sonus21.rqueue.exception.TimedOutException; +import com.github.sonus21.rqueue.test.common.SpringTestBase; import com.github.sonus21.rqueue.test.dto.ChatIndexing; import com.github.sonus21.rqueue.test.dto.Email; import com.github.sonus21.rqueue.test.dto.FeedGeneration; import com.github.sonus21.rqueue.test.dto.Job; -import com.github.sonus21.rqueue.test.dto.Sms; import com.github.sonus21.rqueue.test.dto.Reservation; +import com.github.sonus21.rqueue.test.dto.Sms; import com.github.sonus21.rqueue.utils.Constants; import com.github.sonus21.rqueue.utils.TimeoutUtils; import java.util.Arrays; +import java.util.concurrent.TimeUnit; public abstract class AllQueueMode extends SpringTestBase { protected void checkGroupConsumer() throws TimedOutException { @@ -65,7 +67,7 @@ protected void testSimpleConsumer() throws TimedOutException { rqueueMessageSender.enqueueIn(emailQueue, Email.newInstance(), 1000L); rqueueMessageSender.enqueue(jobQueue, Job.newInstance()); - rqueueMessageSender.enqueueIn(jobQueue, Job.newInstance(), 2000L); + rqueueMessageSender.enqueueIn(jobQueue, Job.newInstance(), 2000L, TimeUnit.MILLISECONDS); TimeoutUtils.waitFor( () -> getMessageCount(Arrays.asList(emailQueue, jobQueue)) == 0, diff --git a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/GroupPriorityListener.java b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/GroupPriorityTest.java similarity index 92% rename from rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/GroupPriorityListener.java rename to rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/GroupPriorityTest.java index 3305c0fd..e39564c9 100644 --- a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/GroupPriorityListener.java +++ b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/GroupPriorityTest.java @@ -17,13 +17,14 @@ package com.github.sonus21.rqueue.test.tests; import com.github.sonus21.rqueue.exception.TimedOutException; +import com.github.sonus21.rqueue.test.common.SpringTestBase; import com.github.sonus21.rqueue.test.dto.ChatIndexing; import com.github.sonus21.rqueue.test.dto.FeedGeneration; import com.github.sonus21.rqueue.test.dto.Reservation; import com.github.sonus21.rqueue.utils.TimeoutUtils; import java.util.Arrays; -public abstract class GroupPriorityListener extends SpringTestBase { +public abstract class GroupPriorityTest extends SpringTestBase { protected void checkGroupConsumer() throws TimedOutException { rqueueMessageSender.enqueue(chatIndexingQueue, ChatIndexing.newInstance()); diff --git a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/MessageChannelTest.java b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/MessageChannelTest.java similarity index 51% rename from rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/MessageChannelTest.java rename to rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/MessageChannelTest.java index a73212cd..581ba2f5 100644 --- a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/MessageChannelTest.java +++ b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/MessageChannelTest.java @@ -14,49 +14,38 @@ * limitations under the License. */ -package com.github.sonus21.rqueue.spring.boot.tests.integration; +package com.github.sonus21.rqueue.test.tests; import static com.github.sonus21.rqueue.utils.TimeoutUtils.waitFor; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import com.github.sonus21.rqueue.exception.TimedOutException; -import com.github.sonus21.rqueue.spring.boot.application.ApplicationListenerDisabled; +import com.github.sonus21.rqueue.test.common.SpringTestBase; import com.github.sonus21.rqueue.test.dto.Email; -import com.github.sonus21.rqueue.test.tests.SpringTestBase; -import com.github.sonus21.test.RqueueSpringTestRunner; +import com.github.sonus21.rqueue.test.dto.Job; +import com.github.sonus21.rqueue.utils.Constants; +import com.github.sonus21.rqueue.utils.TimeoutUtils; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; import lombok.extern.slf4j.Slf4j; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.TestPropertySource; -@RunWith(RqueueSpringTestRunner.class) -@ContextConfiguration(classes = ApplicationListenerDisabled.class) -@TestPropertySource( - properties = { - "rqueue.scheduler.auto.start=false", - "spring.redis.port=8002", - "mysql.db.name=test2" - }) -@SpringBootTest @Slf4j -public class MessageChannelTest extends SpringTestBase { +public abstract class MessageChannelTest extends SpringTestBase { + private int messageCount = 200; /** * This test verified whether any pending message in the delayed queue are moved or not Whenever a * delayed message is pushed then it's checked whether there're any pending messages on delay * queue. if expired delayed messages are found on the head then a message is published on delayed * channel. */ - @Test - public void publishMessageIsTriggeredOnMessageAddition() throws TimedOutException { - int messageCount = 200; + protected void verifyPublishMessageIsTriggeredOnMessageAddition() throws TimedOutException { String delayedQueueName = rqueueConfig.getDelayedQueueName(emailQueue); enqueueIn(delayedQueueName, i -> Email.newInstance(), i -> -1000L, messageCount); Email email = Email.newInstance(); log.info("adding new message {}", email); - messageSender.enqueueIn(emailQueue, email, 1000L); + messageSender.enqueueIn(emailQueue, email, Duration.ofMillis(1000)); waitFor( () -> stringRqueueRedisTemplate.getZsetSize(delayedQueueName) <= 1, "one or zero messages in zset"); @@ -64,6 +53,34 @@ public void publishMessageIsTriggeredOnMessageAddition() throws TimedOutExceptio "Messages are correctly moved", stringRqueueRedisTemplate.getListSize(rqueueConfig.getQueueName(emailQueue)) >= messageCount); - assertEquals(messageCount + 1, messageSender.getAllMessages(emailQueue).size()); + assertEquals(messageCount + 1L, messageSender.getAllMessages(emailQueue).size()); + } + + protected void verifyPublishMessageIsTriggeredOnMessageRemoval() throws TimedOutException { + String processingQueueName = jobQueue; + List jobs = new ArrayList<>(); + List ids = new ArrayList<>(); + int maxDelay = 2000; + for (int i = 0; i < messageCount; i++) { + Job job = Job.newInstance(); + jobs.add(job); + ids.add(job.getId()); + int delay = random.nextInt(maxDelay); + if (random.nextBoolean()) { + delay = delay * -1; + } + enqueueIn(job, processingQueueName, delay); + } + TimeoutUtils.sleep(maxDelay); + waitFor( + () -> 0 == messageSender.getAllMessages(jobQueue).size(), + 30 * Constants.ONE_MILLI, + "messages to be consumed"); + waitFor( + () -> messageCount == consumedMessageService.getMessages(ids, Job.class).size(), + "message count to be matched"); + waitFor( + () -> jobs.containsAll(consumedMessageService.getMessages(ids, Job.class).values()), + "All jobs to be executed"); } } diff --git a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/MessageRetryTest.java b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/MessageRetryTest.java new file mode 100644 index 00000000..ba32bfb3 --- /dev/null +++ b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/MessageRetryTest.java @@ -0,0 +1,129 @@ +/* + * Copyright 2020 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.sonus21.rqueue.test.tests; + +import static com.github.sonus21.rqueue.utils.TimeoutUtils.waitFor; +import static org.junit.Assert.assertEquals; + +import com.github.sonus21.rqueue.exception.TimedOutException; +import com.github.sonus21.rqueue.test.common.SpringTestBase; +import com.github.sonus21.rqueue.test.dto.Email; +import com.github.sonus21.rqueue.test.dto.Job; +import com.github.sonus21.rqueue.test.dto.Notification; +import java.util.List; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public abstract class MessageRetryTest extends SpringTestBase { + protected void verifyAfterNRetryTaskIsDeletedFromProcessingQueue() throws TimedOutException { + cleanQueue(jobQueue); + Job job = Job.newInstance(); + failureManager.createFailureDetail(job.getId(), 3, 10); + messageSender.put(jobQueue, job); + waitFor( + () -> { + Job jobInDb = consumedMessageService.getMessage(job.getId(), Job.class); + return job.equals(jobInDb); + }, + "job to be executed"); + waitFor( + () -> { + List messages = messageSender.getAllMessages(jobQueue); + return !messages.contains(job); + }, + "message should be deleted from internal storage"); + } + + protected void verifyMessageMovedToDelayedQueue() throws TimedOutException { + cleanQueue(emailQueue); + Email email = Email.newInstance(); + failureManager.createFailureDetail(email.getId(), -1, 0); + log.debug("queue: {} msg: {}", emailQueue, email); + messageSender.put(emailQueue, email, 1000L); + waitFor( + () -> emailRetryCount == failureManager.getFailureCount(email.getId()), + 30000000, + "all retry to be exhausted"); + waitFor( + () -> stringRqueueRedisTemplate.getListSize(emailDeadLetterQueue) > 0, + "message should be moved to delayed queue"); + assertEquals(emailRetryCount, failureManager.getFailureCount(email.getId())); + failureManager.delete(email.getId()); + } + + protected void verifyMessageIsDiscardedAfterRetries() throws TimedOutException { + cleanQueue(notificationQueue); + Notification notification = Notification.newInstance(); + failureManager.createFailureDetail(notification.getId(), -1, notificationRetryCount); + messageSender.enqueueAt(notificationQueue, notification, System.currentTimeMillis() + 1000L); + waitFor( + () -> notificationRetryCount == failureManager.getFailureCount(notification.getId()), + "all retry to be exhausted"); + waitFor( + () -> { + List messages = messageSender.getAllMessages(notificationQueue); + return !messages.contains(notification); + }, + "message to be discarded"); + assertEquals(notificationRetryCount, failureManager.getFailureCount(notification.getId())); + } + + public void verifySimpleTaskExecution() throws TimedOutException { + cleanQueue(notificationQueue); + Notification notification = Notification.newInstance(); + messageSender.enqueue(notificationQueue, notification); + waitFor( + () -> { + Notification notificationInDb = + consumedMessageService.getMessage(notification.getId(), Notification.class); + return notification.equals(notificationInDb); + }, + "notification to be executed"); + } + + public void verifyRetry() throws TimedOutException { + cleanQueue(emailQueue); + Email email = Email.newInstance(); + failureManager.createFailureDetail(email.getId(), emailRetryCount - 1, emailRetryCount - 1); + messageSender.enqueue(emailQueue, email); + waitFor( + () -> failureManager.getFailureCount(email.getId()) == emailRetryCount - 1, + "email task needs to be retried"); + waitFor( + () -> { + Email emailInDb = consumedMessageService.getMessage(email.getId(), Email.class); + return email.equals(emailInDb); + }, + "job to be executed"); + } + + public void verifyMessageIsInProcessingQueue() throws TimedOutException { + cleanQueue(jobQueue); + Job job = Job.newInstance(); + failureManager.createFailureDetail(job.getId(), -1, 0); + messageSender.enqueue(jobQueue, job); + waitFor(() -> failureManager.getFailureCount(job.getId()) >= 3, "Job to be retried"); + waitFor( + () -> { + List messages = messageSender.getAllMessages(jobQueue); + return messages.contains(job); + }, + "message should be present in internal storage"); + // more then one copy should not be present + assertEquals(1, messageSender.getAllMessages(jobQueue).size()); + } +} diff --git a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/MetricTestBase.java b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/MetricTest.java similarity index 93% rename from rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/MetricTestBase.java rename to rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/MetricTest.java index ddbd182b..4a26dbd8 100644 --- a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/MetricTestBase.java +++ b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/MetricTest.java @@ -22,16 +22,17 @@ import static org.junit.Assert.assertTrue; import com.github.sonus21.rqueue.exception.TimedOutException; +import com.github.sonus21.rqueue.test.common.SpringTestBase; import com.github.sonus21.rqueue.test.dto.Email; import com.github.sonus21.rqueue.test.dto.Job; import com.github.sonus21.rqueue.test.dto.Notification; +import com.github.sonus21.rqueue.utils.Constants; import io.micrometer.core.instrument.MeterRegistry; -import java.util.Random; +import java.time.Instant; import org.springframework.beans.factory.annotation.Autowired; -public abstract class MetricTestBase extends SpringTestBase { +public abstract class MetricTest extends SpringTestBase { @Autowired protected MeterRegistry meterRegistry; - private Random random = new Random(); protected void verifyDelayedQueueStatus() throws TimedOutException { long maxDelay = 0; @@ -45,7 +46,7 @@ protected void verifyDelayedQueueStatus() throws TimedOutException { if (i < maxMessages / 2) { enqueueIn(notification, rqueueConfig.getDelayedQueueName(notificationQueue), -delay); } else { - messageSender.enqueueIn(notificationQueue, notification, delay); + messageSender.enqueueAt(notificationQueue, notification, Instant.now().plusMillis(delay)); } } @@ -103,6 +104,7 @@ protected void verifyMetricStatus() throws TimedOutException { .gauge() .value() == 1, + 30 * Constants.ONE_MILLI, "processing queue message"); } diff --git a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/MultiLevelQueueListenerTestBase.java b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/MultiLevelQueueTest.java similarity index 59% rename from rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/MultiLevelQueueListenerTestBase.java rename to rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/MultiLevelQueueTest.java index 6df59042..45364185 100644 --- a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/MultiLevelQueueListenerTestBase.java +++ b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/tests/MultiLevelQueueTest.java @@ -17,11 +17,14 @@ package com.github.sonus21.rqueue.test.tests; import com.github.sonus21.rqueue.exception.TimedOutException; +import com.github.sonus21.rqueue.test.common.SpringTestBase; import com.github.sonus21.rqueue.test.dto.Sms; import com.github.sonus21.rqueue.utils.TimeoutUtils; +import java.time.Duration; import java.util.Arrays; +import java.util.concurrent.TimeUnit; -public abstract class MultiLevelQueueListenerTestBase extends SpringTestBase { +public abstract class MultiLevelQueueTest extends SpringTestBase { protected void checkQueueLevelConsumer() throws TimedOutException { rqueueMessageSender.enqueue(smsQueue, Sms.newInstance()); rqueueMessageSender.enqueueWithPriority(smsQueue, "critical", Sms.newInstance()); @@ -40,4 +43,25 @@ protected void checkQueueLevelConsumer() throws TimedOutException { == 0, "Waiting for multi level queues to drain"); } + + protected void checkQueueLevelConsumerWithDelay() throws TimedOutException { + rqueueMessageSender.enqueue(smsQueue, Sms.newInstance()); + rqueueMessageSender.enqueueInWithPriority(smsQueue, "critical", Sms.newInstance(), 1000L); + rqueueMessageSender.enqueueInWithPriority( + smsQueue, "high", Sms.newInstance(), 1000L, TimeUnit.MILLISECONDS); + rqueueMessageSender.enqueueInWithPriority( + smsQueue, "medium", Sms.newInstance(), Duration.ofMillis(1000L)); + rqueueMessageSender.enqueueInWithPriority(smsQueue, "low", Sms.newInstance(), 1000L); + TimeoutUtils.waitFor( + () -> + getMessageCount( + Arrays.asList( + smsQueue, + smsQueue + "_critical", + smsQueue + "_high", + smsQueue + "_medium", + smsQueue + "_low")) + == 0, + "Waiting for multi level queues to drain"); + } } diff --git a/rqueue-common-test/src/main/resources/logback.xml b/rqueue-common-test/src/main/resources/logback.xml index 5c4eeab3..1f525748 100644 --- a/rqueue-common-test/src/main/resources/logback.xml +++ b/rqueue-common-test/src/main/resources/logback.xml @@ -10,7 +10,7 @@ log/test.log - log/test.%d{yyyy-MM-dd-HH-mm}.log + log/test.%d{yyyy-MM-dd-HH}.log 30 100MB diff --git a/rqueue-spring-boot-example/build.gradle b/rqueue-spring-boot-example/build.gradle index 213ba53a..5592c7c6 100644 --- a/rqueue-spring-boot-example/build.gradle +++ b/rqueue-spring-boot-example/build.gradle @@ -6,6 +6,9 @@ plugins { dependencies { compile project(":rqueue-spring-boot-starter") compile project(":rqueue-common-test") + // https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-devtools + compile group: 'org.springframework.boot', name: 'spring-boot-devtools', version: '2.1.0.RELEASE' + // https://mvnrepository.com/artifact/org.springframework.cloud/spring-cloud-starter-sleuth compile group: 'org.springframework.cloud', name: 'spring-cloud-starter-sleuth', version: '2.1.0.RELEASE' compile 'org.springframework.boot:spring-boot-starter-data-redis' diff --git a/rqueue-spring-boot-example/src/main/java/com/github/sonus21/rqueue/example/Controller.java b/rqueue-spring-boot-example/src/main/java/com/github/sonus21/rqueue/example/Controller.java index df6808dd..e6f650a7 100644 --- a/rqueue-spring-boot-example/src/main/java/com/github/sonus21/rqueue/example/Controller.java +++ b/rqueue-spring-boot-example/src/main/java/com/github/sonus21/rqueue/example/Controller.java @@ -19,8 +19,7 @@ import com.github.sonus21.rqueue.core.RqueueMessageSender; import java.util.UUID; import lombok.AllArgsConstructor; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestParam; @@ -28,9 +27,9 @@ @RestController @AllArgsConstructor(onConstructor = @__(@Autowired)) +@Slf4j public class Controller { private RqueueMessageSender rqueueMessageSender; - private static Logger log = LoggerFactory.getLogger(Controller.class); @GetMapping(value = "/push") public String push( diff --git a/rqueue-spring-boot-example/src/main/resources/logback.xml b/rqueue-spring-boot-example/src/main/resources/logback.xml index 673a3c74..eff8a671 100644 --- a/rqueue-spring-boot-example/src/main/resources/logback.xml +++ b/rqueue-spring-boot-example/src/main/resources/logback.xml @@ -4,6 +4,17 @@ [%date{dd-MM-yyyy HH:mm:ss.SSS}] [%thread] %-5level %X{traceId:-} %X{spanId:-} ${PID:-} %logger{36} - %msg%n + + + [%date{dd-MM-yyyy HH:mm:ss.SSS}] [%thread] %-5level %X{traceId:-} %X{spanId:-} ${PID:-} %logger{36} - %msg%n + + log/app.log + + log/app.%d{yyyy-MM-dd-HH}.log + 30 + 200MB + + diff --git a/rqueue-spring-boot-starter/src/main/java/com/github/sonus21/rqueue/spring/boot/RqueueListenerAutoConfig.java b/rqueue-spring-boot-starter/src/main/java/com/github/sonus21/rqueue/spring/boot/RqueueListenerAutoConfig.java index 8f5ac426..3d1971bc 100644 --- a/rqueue-spring-boot-starter/src/main/java/com/github/sonus21/rqueue/spring/boot/RqueueListenerAutoConfig.java +++ b/rqueue-spring-boot-starter/src/main/java/com/github/sonus21/rqueue/spring/boot/RqueueListenerAutoConfig.java @@ -18,11 +18,11 @@ import com.github.sonus21.rqueue.config.RqueueConfig; import com.github.sonus21.rqueue.config.RqueueListenerBaseConfig; +import com.github.sonus21.rqueue.core.RqueueMessageSender; +import com.github.sonus21.rqueue.core.RqueueMessageSenderImpl; import com.github.sonus21.rqueue.core.RqueueMessageTemplate; import com.github.sonus21.rqueue.listener.RqueueMessageHandler; import com.github.sonus21.rqueue.listener.RqueueMessageListenerContainer; -import com.github.sonus21.rqueue.core.RqueueMessageSender; -import com.github.sonus21.rqueue.core.RqueueMessageSenderImpl; import org.springframework.boot.autoconfigure.AutoConfigureAfter; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration; diff --git a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/application/Application.java b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/application/Application.java index 75788ab2..c7c24ee0 100644 --- a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/application/Application.java +++ b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/application/Application.java @@ -16,7 +16,7 @@ package com.github.sonus21.rqueue.spring.boot.application; -import com.github.sonus21.rqueue.test.BaseApplication; +import com.github.sonus21.rqueue.test.application.BaseApplication; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.context.annotation.PropertySource; diff --git a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/application/ApplicationListenerDisabled.java b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/application/ApplicationListenerDisabled.java index 4ceff67a..abdefef4 100644 --- a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/application/ApplicationListenerDisabled.java +++ b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/application/ApplicationListenerDisabled.java @@ -21,7 +21,7 @@ import com.github.sonus21.rqueue.listener.QueueDetail; import com.github.sonus21.rqueue.listener.RqueueMessageHandler; import com.github.sonus21.rqueue.listener.RqueueMessageListenerContainer; -import com.github.sonus21.rqueue.test.BaseApplication; +import com.github.sonus21.rqueue.test.application.BaseApplication; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.context.annotation.Bean; diff --git a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/application/ApplicationWithCustomConfiguration.java b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/application/ApplicationWithCustomConfiguration.java index 58237829..f5a2ff6b 100644 --- a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/application/ApplicationWithCustomConfiguration.java +++ b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/application/ApplicationWithCustomConfiguration.java @@ -20,7 +20,7 @@ import com.github.sonus21.rqueue.core.RqueueMessageTemplateImpl; import com.github.sonus21.rqueue.listener.RqueueMessageHandler; import com.github.sonus21.rqueue.listener.RqueueMessageListenerContainer; -import com.github.sonus21.rqueue.test.BaseApplication; +import com.github.sonus21.rqueue.test.application.BaseApplication; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; diff --git a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/application/MultiRedisSetupApplication.java b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/application/MultiRedisSetupApplication.java new file mode 100644 index 00000000..c19a6507 --- /dev/null +++ b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/application/MultiRedisSetupApplication.java @@ -0,0 +1,36 @@ +/* + * Copyright 2020 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.sonus21.rqueue.spring.boot.application; + +import com.github.sonus21.rqueue.test.application.MultiRedisSprigBaseApplication; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.annotation.PropertySource; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; +import org.springframework.data.redis.repository.configuration.EnableRedisRepositories; +import org.springframework.transaction.annotation.EnableTransactionManagement; + +@PropertySource("classpath:application.properties") +@SpringBootApplication(scanBasePackages = {"com.github.sonus21.rqueue.test"}) +@EnableRedisRepositories +@EnableJpaRepositories(basePackages = {"com.github.sonus21.rqueue.test.repository"}) +@EnableTransactionManagement +public class MultiRedisSetupApplication extends MultiRedisSprigBaseApplication { + public static void main(String[] args) { + SpringApplication.run(MultiRedisSetupApplication.class, args); + } +} diff --git a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/application/RedisClusterApplication.java b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/application/RedisClusterApplication.java new file mode 100644 index 00000000..5b7e6672 --- /dev/null +++ b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/application/RedisClusterApplication.java @@ -0,0 +1,36 @@ +/* + * Copyright 2020 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.sonus21.rqueue.spring.boot.application; + +import com.github.sonus21.rqueue.test.application.RedisClusterBaseApplication; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.annotation.PropertySource; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; +import org.springframework.data.redis.repository.configuration.EnableRedisRepositories; +import org.springframework.transaction.annotation.EnableTransactionManagement; + +@PropertySource("classpath:application.properties") +@SpringBootApplication(scanBasePackages = {"com.github.sonus21.rqueue.test"}) +@EnableRedisRepositories +@EnableJpaRepositories(basePackages = {"com.github.sonus21.rqueue.test.repository"}) +@EnableTransactionManagement +public class RedisClusterApplication extends RedisClusterBaseApplication { + public static void main(String[] args) { + SpringApplication.run(RedisClusterApplication.class, args); + } +} diff --git a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/ApplicationTest.java b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/ApplicationTest.java index 6f50333b..bc7afe00 100644 --- a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/ApplicationTest.java +++ b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/ApplicationTest.java @@ -16,17 +16,10 @@ package com.github.sonus21.rqueue.spring.boot.tests.integration; -import static com.github.sonus21.rqueue.utils.TimeoutUtils.waitFor; -import static org.junit.Assert.assertEquals; - import com.github.sonus21.rqueue.exception.TimedOutException; import com.github.sonus21.rqueue.spring.boot.application.Application; -import com.github.sonus21.rqueue.test.dto.Email; -import com.github.sonus21.rqueue.test.dto.Job; -import com.github.sonus21.rqueue.test.dto.Notification; -import com.github.sonus21.rqueue.test.tests.SpringTestBase; +import com.github.sonus21.rqueue.test.tests.MessageRetryTest; import com.github.sonus21.test.RqueueSpringTestRunner; -import java.util.List; import lombok.extern.slf4j.Slf4j; import org.junit.Test; import org.junit.runner.RunWith; @@ -39,58 +32,35 @@ @SpringBootTest @Slf4j @TestPropertySource(properties = {"rqueue.retry.per.poll=1000", "spring.redis.port=8001"}) -public class ApplicationTest extends SpringTestBase { +public class ApplicationTest extends MessageRetryTest { @Test public void afterNRetryTaskIsDeletedFromProcessingQueue() throws TimedOutException { - Job job = Job.newInstance(); - failureManager.createFailureDetail(job.getId(), 3, 10); - messageSender.put(jobQueue, job); - waitFor( - () -> { - Job jobInDb = consumedMessageService.getMessage(job.getId(), Job.class); - return job.equals(jobInDb); - }, - "job to be executed"); - waitFor( - () -> { - List messages = messageSender.getAllMessages(jobQueue); - return !messages.contains(job); - }, - "message should be deleted from internal storage"); + verifyAfterNRetryTaskIsDeletedFromProcessingQueue(); } @Test public void messageMovedToDelayedQueue() throws TimedOutException { - Email email = Email.newInstance(); - failureManager.createFailureDetail(email.getId(), -1, 0); - log.debug("queue: {} msg: {}", emailQueue, email); - messageSender.put(emailQueue, email, 1000L); - waitFor( - () -> emailRetryCount == failureManager.getFailureCount(email.getId()), - 30000000, - "all retry to be exhausted"); - waitFor( - () -> stringRqueueRedisTemplate.getListSize(emailDeadLetterQueue) > 0, - "message should be moved to delayed queue"); - assertEquals(emailRetryCount, failureManager.getFailureCount(email.getId())); - failureManager.delete(email.getId()); + verifyMessageMovedToDelayedQueue(); } @Test public void messageIsDiscardedAfterRetries() throws TimedOutException { - Notification notification = Notification.newInstance(); - failureManager.createFailureDetail(notification.getId(), -1, notificationRetryCount); - messageSender.enqueueIn(notificationQueue, notification, 1000L); - waitFor( - () -> notificationRetryCount == failureManager.getFailureCount(notification.getId()), - "all retry to be exhausted"); - waitFor( - () -> { - List messages = messageSender.getAllMessages(notificationQueue); - return !messages.contains(notification); - }, - "message to be discarded"); - assertEquals(notificationRetryCount, failureManager.getFailureCount(notification.getId())); + verifyMessageIsDiscardedAfterRetries(); + } + + @Test + public void notificationOnMessageIsTriggered() throws TimedOutException { + verifySimpleTaskExecution(); + } + + @Test + public void emailIsRetried() throws TimedOutException { + verifyRetry(); + } + + @Test + public void jobIsRetriedAndMessageIsInProcessingQueue() throws TimedOutException { + verifyMessageIsInProcessingQueue(); } } diff --git a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/ProcessingMessageSchedulerTest.java b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/BootMessageChannelTest.java similarity index 56% rename from rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/ProcessingMessageSchedulerTest.java rename to rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/BootMessageChannelTest.java index 1f0e7d42..cf6df791 100644 --- a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/ProcessingMessageSchedulerTest.java +++ b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/BootMessageChannelTest.java @@ -16,21 +16,14 @@ package com.github.sonus21.rqueue.spring.boot.tests.integration; -import static com.github.sonus21.rqueue.utils.TimeoutUtils.waitFor; - import com.github.sonus21.rqueue.core.RqueueMessage; import com.github.sonus21.rqueue.exception.TimedOutException; -import com.github.sonus21.rqueue.spring.boot.application.ApplicationWithCustomConfiguration; -import com.github.sonus21.rqueue.test.dto.Job; -import com.github.sonus21.rqueue.test.tests.SpringTestBase; -import com.github.sonus21.rqueue.utils.Constants; -import com.github.sonus21.rqueue.utils.TimeoutUtils; +import com.github.sonus21.rqueue.spring.boot.application.ApplicationListenerDisabled; +import com.github.sonus21.rqueue.test.tests.MessageChannelTest; import com.github.sonus21.test.RqueueSpringTestRunner; import com.github.sonus21.test.RunTestUntilFail; -import java.util.ArrayList; import java.util.List; import java.util.Map.Entry; -import java.util.Random; import lombok.extern.slf4j.Slf4j; import org.junit.Rule; import org.junit.Test; @@ -39,19 +32,21 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; -@SpringBootTest @RunWith(RqueueSpringTestRunner.class) -@ContextConfiguration(classes = ApplicationWithCustomConfiguration.class) -@Slf4j +@ContextConfiguration(classes = ApplicationListenerDisabled.class) @TestPropertySource( properties = { "rqueue.scheduler.auto.start=false", - "spring.redis.port=8005", - "mysql.db.name=test3", + "spring.redis.port=8002", + "mysql.db.name=test2", "max.workers.count=120", "use.system.redis=false" + }) -public class ProcessingMessageSchedulerTest extends SpringTestBase { +@SpringBootTest +@Slf4j +public class BootMessageChannelTest extends MessageChannelTest { + @Rule public RunTestUntilFail retry = new RunTestUntilFail( @@ -65,36 +60,13 @@ public class ProcessingMessageSchedulerTest extends SpringTestBase { } }); - private int messageCount = 110; + @Test + public void publishMessageIsTriggeredOnMessageAddition() throws TimedOutException { + verifyPublishMessageIsTriggeredOnMessageAddition(); + } @Test public void publishMessageIsTriggeredOnMessageRemoval() throws TimedOutException { - String processingQueueName = jobQueue; - long currentTime = System.currentTimeMillis(); - List jobs = new ArrayList<>(); - List ids = new ArrayList<>(); - int maxDelay = 2000; - Random random = new Random(); - for (int i = 0; i < messageCount; i++) { - Job job = Job.newInstance(); - jobs.add(job); - ids.add(job.getId()); - int delay = random.nextInt(maxDelay); - if (random.nextBoolean()) { - delay = delay * -1; - } - enqueueIn(job, processingQueueName, delay); - } - TimeoutUtils.sleep(maxDelay); - waitFor( - () -> 0 == messageSender.getAllMessages(jobQueue).size(), - 30 * Constants.ONE_MILLI, - "messages to be consumed"); - waitFor( - () -> messageCount == consumedMessageService.getMessages(ids, Job.class).size(), - "message count to be matched"); - waitFor( - () -> jobs.containsAll(consumedMessageService.getMessages(ids, Job.class).values()), - "All jobs to be executed"); + verifyPublishMessageIsTriggeredOnMessageRemoval(); } } diff --git a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/BootMetricsTest.java b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/BootMetricsTest.java index 64889c90..547b07f4 100644 --- a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/BootMetricsTest.java +++ b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/BootMetricsTest.java @@ -18,7 +18,7 @@ import com.github.sonus21.rqueue.exception.TimedOutException; import com.github.sonus21.rqueue.spring.boot.application.Application; -import com.github.sonus21.rqueue.test.tests.MetricTestBase; +import com.github.sonus21.rqueue.test.tests.MetricTest; import com.github.sonus21.test.RqueueSpringTestRunner; import lombok.extern.slf4j.Slf4j; import org.junit.Test; @@ -35,12 +35,12 @@ properties = { "rqueue.retry.per.poll=20", "rqueue.scheduler.auto.start=false", - "spring.redis.port=8004", + "spring.redis.port=8003", "mysql.db.name=test4", "rqueue.metrics.count.failure=true", "rqueue.metrics.count.execution=true", }) -public class BootMetricsTest extends MetricTestBase { +public class BootMetricsTest extends MetricTest { @Test public void delayedQueueStatus() throws TimedOutException { diff --git a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/MessageRetryTest.java b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/MessageRetryTest.java deleted file mode 100644 index 576dd853..00000000 --- a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/MessageRetryTest.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright 2020 Sonu Kumar - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.github.sonus21.rqueue.spring.boot.tests.integration; - -import static com.github.sonus21.rqueue.utils.TimeoutUtils.waitFor; -import static org.junit.Assert.assertEquals; - -import com.github.sonus21.rqueue.exception.TimedOutException; -import com.github.sonus21.rqueue.spring.boot.application.Application; -import com.github.sonus21.rqueue.test.dto.Email; -import com.github.sonus21.rqueue.test.dto.Job; -import com.github.sonus21.rqueue.test.dto.Notification; -import com.github.sonus21.rqueue.test.tests.SpringTestBase; -import com.github.sonus21.test.RqueueSpringTestRunner; -import java.util.List; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.TestPropertySource; - -@RunWith(RqueueSpringTestRunner.class) -@ContextConfiguration(classes = Application.class) -@TestPropertySource( - properties = {"spring.redis.port=8003", "mysql.db.name=test1", "rqueue.retry.per.poll=1000"}) -@SpringBootTest -public class MessageRetryTest extends SpringTestBase { - @Test - public void notificationOnMessageIsTriggered() throws TimedOutException { - Notification notification = Notification.newInstance(); - messageSender.enqueue(notificationQueue, notification); - waitFor( - () -> { - Notification notificationInDb = - consumedMessageService.getMessage(notification.getId(), Notification.class); - return notification.equals(notificationInDb); - }, - "notification to be executed"); - } - - @Test - public void emailIsRetried() throws TimedOutException { - Email email = Email.newInstance(); - failureManager.createFailureDetail(email.getId(), emailRetryCount - 1, emailRetryCount - 1); - messageSender.enqueue(emailQueue, email); - waitFor( - () -> failureManager.getFailureCount(email.getId()) == emailRetryCount - 1, - "email task needs to be retried"); - waitFor( - () -> { - Email emailInDb = consumedMessageService.getMessage(email.getId(), Email.class); - return email.equals(emailInDb); - }, - "job to be executed"); - } - - @Test - public void jobIsRetriedAndMessageIsInProcessingQueue() throws TimedOutException { - Job job = Job.newInstance(); - failureManager.createFailureDetail(job.getId(), -1, 0); - messageSender.enqueue(jobQueue, job); - waitFor(() -> failureManager.getFailureCount(job.getId()) >= 3, "Job to be retried"); - waitFor( - () -> { - List messages = messageSender.getAllMessages(jobQueue); - return messages.contains(job); - }, - "message should be present in internal storage"); - // more then one copy should not be present - assertEquals(1, messageSender.getAllMessages(jobQueue).size()); - } -} diff --git a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/MultiRedisSetup.java b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/MultiRedisSetup.java new file mode 100644 index 00000000..7eed2147 --- /dev/null +++ b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/MultiRedisSetup.java @@ -0,0 +1,57 @@ +/* + * Copyright 2020 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.sonus21.rqueue.spring.boot.tests.integration; + +import com.github.sonus21.rqueue.exception.TimedOutException; +import com.github.sonus21.rqueue.spring.boot.application.MultiRedisSetupApplication; +import com.github.sonus21.rqueue.test.tests.MessageRetryTest; +import com.github.sonus21.test.RqueueSpringTestRunner; +import lombok.extern.slf4j.Slf4j; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.TestPropertySource; + +@RunWith(RqueueSpringTestRunner.class) +@ContextConfiguration(classes = MultiRedisSetupApplication.class) +@SpringBootTest +@Slf4j +@TestPropertySource( + properties = { + "rqueue.retry.per.poll=1000", + "spring.redis.port=8005", + "spring.redis2.port=8006", + "spring.redis2.host=localhost" + }) +public class MultiRedisSetup extends MessageRetryTest { + + @Test + public void afterNRetryTaskIsDeletedFromProcessingQueue() throws TimedOutException { + verifyAfterNRetryTaskIsDeletedFromProcessingQueue(); + } + + @Test + public void messageMovedToDelayedQueue() throws TimedOutException { + verifyMessageMovedToDelayedQueue(); + } + + @Test + public void messageIsDiscardedAfterRetries() throws TimedOutException { + verifyMessageIsDiscardedAfterRetries(); + } +} diff --git a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/RedisClusterTest.java b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/RedisClusterTest.java new file mode 100644 index 00000000..833b23c4 --- /dev/null +++ b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/RedisClusterTest.java @@ -0,0 +1,51 @@ +/* + * Copyright 2020 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.sonus21.rqueue.spring.boot.tests.integration; + +import com.github.sonus21.rqueue.exception.TimedOutException; +import com.github.sonus21.rqueue.spring.boot.application.RedisClusterApplication; +import com.github.sonus21.rqueue.test.tests.MessageRetryTest; +import com.github.sonus21.test.RqueueSpringTestRunner; +import lombok.extern.slf4j.Slf4j; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.TestPropertySource; + +@RunWith(RqueueSpringTestRunner.class) +@ContextConfiguration(classes = RedisClusterApplication.class) +@SpringBootTest +@Slf4j +@TestPropertySource(properties = {"rqueue.retry.per.poll=1000", "spring.redis.port=8007"}) +public class RedisClusterTest extends MessageRetryTest { + + @Test + public void afterNRetryTaskIsDeletedFromProcessingQueue() throws TimedOutException { + verifyAfterNRetryTaskIsDeletedFromProcessingQueue(); + } + + @Test + public void messageMovedToDelayedQueue() throws TimedOutException { + verifyMessageMovedToDelayedQueue(); + } + + @Test + public void messageIsDiscardedAfterRetries() throws TimedOutException { + verifyMessageIsDiscardedAfterRetries(); + } +} diff --git a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/RqueueMessageTemplateTest.java b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/RqueueMessageTemplateTest.java index f1d7eaf9..8d398f35 100644 --- a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/RqueueMessageTemplateTest.java +++ b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/RqueueMessageTemplateTest.java @@ -20,7 +20,7 @@ import com.github.sonus21.rqueue.test.dto.Email; import com.github.sonus21.rqueue.test.dto.Job; import com.github.sonus21.rqueue.test.dto.Notification; -import com.github.sonus21.rqueue.test.tests.SpringTestBase; +import com.github.sonus21.rqueue.test.common.SpringTestBase; import com.github.sonus21.test.RqueueSpringTestRunner; import lombok.extern.slf4j.Slf4j; import org.junit.Assert; @@ -34,7 +34,7 @@ @ContextConfiguration(classes = Application.class) @SpringBootTest @Slf4j -@TestPropertySource(properties = {"use.system.redis=false", "spring.redis.port:8006"}) +@TestPropertySource(properties = {"use.system.redis=false", "spring.redis.port:8004"}) public class RqueueMessageTemplateTest extends SpringTestBase { @Test public void moveMessageFromDeadLetterQueueToOriginalQueue() { diff --git a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/unit/RqueueListenerAutoConfigTest.java b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/unit/RqueueListenerAutoConfigTest.java index 65c0fac3..cb392644 100644 --- a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/unit/RqueueListenerAutoConfigTest.java +++ b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/unit/RqueueListenerAutoConfigTest.java @@ -23,9 +23,9 @@ import com.github.sonus21.rqueue.config.SimpleRqueueListenerContainerFactory; import com.github.sonus21.rqueue.converter.GenericMessageConverter; +import com.github.sonus21.rqueue.core.RqueueMessageSender; import com.github.sonus21.rqueue.core.RqueueMessageTemplate; import com.github.sonus21.rqueue.listener.RqueueMessageHandler; -import com.github.sonus21.rqueue.core.RqueueMessageSender; import com.github.sonus21.rqueue.spring.boot.RqueueListenerAutoConfig; import java.util.ArrayList; import java.util.Collections; @@ -73,16 +73,6 @@ public void rqueueMessageHandlerReused() throws IllegalAccessException { rqueueMessageHandler.hashCode(), messageAutoConfig.rqueueMessageHandler().hashCode()); } - @Test - public void rqueueMessageHandlerCreatedWithMessageConverters() throws IllegalAccessException { - SimpleRqueueListenerContainerFactory factory = new SimpleRqueueListenerContainerFactory(); - factory.setMessageConverters(messageConverterList); - RqueueListenerAutoConfig messageAutoConfig = new RqueueListenerAutoConfig(); - FieldUtils.writeField(messageAutoConfig, "simpleRqueueListenerContainerFactory", factory, true); - RqueueMessageHandler messageHandler = messageAutoConfig.rqueueMessageHandler(); - assertEquals(messageConverterList.get(0), messageHandler.getMessageConverters().get(0)); - } - @Test public void rqueueMessageListenerContainer() throws IllegalAccessException { SimpleRqueueListenerContainerFactory factory = new SimpleRqueueListenerContainerFactory(); diff --git a/rqueue-spring/src/main/java/com/github/sonus21/rqueue/spring/RqueueListenerConfig.java b/rqueue-spring/src/main/java/com/github/sonus21/rqueue/spring/RqueueListenerConfig.java index 1504d28f..dd8debcf 100644 --- a/rqueue-spring/src/main/java/com/github/sonus21/rqueue/spring/RqueueListenerConfig.java +++ b/rqueue-spring/src/main/java/com/github/sonus21/rqueue/spring/RqueueListenerConfig.java @@ -19,14 +19,14 @@ import com.github.sonus21.rqueue.common.RqueueRedisTemplate; import com.github.sonus21.rqueue.config.RqueueConfig; import com.github.sonus21.rqueue.config.RqueueListenerBaseConfig; +import com.github.sonus21.rqueue.core.RqueueMessageSender; +import com.github.sonus21.rqueue.core.RqueueMessageSenderImpl; import com.github.sonus21.rqueue.core.RqueueMessageTemplate; import com.github.sonus21.rqueue.listener.RqueueMessageHandler; import com.github.sonus21.rqueue.listener.RqueueMessageListenerContainer; import com.github.sonus21.rqueue.metrics.QueueCounter; import com.github.sonus21.rqueue.metrics.RqueueCounter; import com.github.sonus21.rqueue.metrics.RqueueMetrics; -import com.github.sonus21.rqueue.core.RqueueMessageSender; -import com.github.sonus21.rqueue.core.RqueueMessageSenderImpl; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.ComponentScan; diff --git a/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/app/SpringApp.java b/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/app/SpringApp.java index dc7e2e2c..3017d88a 100644 --- a/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/app/SpringApp.java +++ b/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/app/SpringApp.java @@ -21,7 +21,7 @@ import com.github.sonus21.rqueue.models.enums.PriorityMode; import com.github.sonus21.rqueue.spring.EnableRqueue; import com.github.sonus21.rqueue.spring.RqueueMetricsProperties; -import com.github.sonus21.rqueue.test.BaseApplication; +import com.github.sonus21.rqueue.test.application.BaseApplication; import io.micrometer.core.instrument.Tags; import io.micrometer.prometheus.PrometheusConfig; import io.micrometer.prometheus.PrometheusMeterRegistry; @@ -65,24 +65,6 @@ public RqueueMetricsProperties rqueueMetricsProperties() { return metricsProperties; } - public class DeleteMessageListener implements MessageProcessor { - private List messages = new ArrayList<>(); - - @Override - public boolean process(Object message) { - messages.add(message); - return true; - } - - public List getMessages() { - return messages; - } - - public void clear() { - this.messages = new ArrayList<>(); - } - } - @Bean public DeleteMessageListener deleteMessageListener() { return new DeleteMessageListener(); @@ -105,4 +87,22 @@ public SimpleRqueueListenerContainerFactory simpleRqueueListenerContainerFactory } return factory; } + + public class DeleteMessageListener implements MessageProcessor { + private List messages = new ArrayList<>(); + + @Override + public boolean process(Object message) { + messages.add(message); + return true; + } + + public List getMessages() { + return messages; + } + + public void clear() { + this.messages = new ArrayList<>(); + } + } } diff --git a/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/tests/integration/DefaultListenerGroup.java b/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/tests/integration/DefaultListenerGroup.java new file mode 100644 index 00000000..9fcaefb2 --- /dev/null +++ b/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/tests/integration/DefaultListenerGroup.java @@ -0,0 +1,70 @@ +/* + * Copyright 2020 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.sonus21.rqueue.spring.tests.integration; + +import com.github.sonus21.rqueue.exception.TimedOutException; +import com.github.sonus21.rqueue.spring.app.SpringApp; +import com.github.sonus21.rqueue.test.tests.AllQueueMode; +import com.github.sonus21.test.RqueueSpringTestRunner; +import lombok.extern.slf4j.Slf4j; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.web.WebAppConfiguration; + +@ContextConfiguration(classes = SpringApp.class) +@RunWith(RqueueSpringTestRunner.class) +@Slf4j +@WebAppConfiguration +@TestPropertySource( + properties = { + "spring.redis.port=7015", + "mysql.db.name=DefaultListenerGroup", + "sms.queue.active=true", + "sms.queue.group=sms-test", + "notification.queue.active=false", + "email.queue.active=true", + "job.queue.active=true", + "priority.mode=STRICT", + "reservation.queue.active=true", + "reservation.queue.group=", + "feed.generation.queue.active=true", + "feed.generation.queue.group=", + "chat.indexing.queue.active=true", + "chat.indexing.queue.group=", + "sms.queue.concurrency=5", + "reservation.queue.concurrency=2", + "feed.generation.queue.concurrency=1-5", + "chat.indexing.queue.concurrency=3-5" + }) +public class DefaultListenerGroup extends AllQueueMode { + @Test + public void verifySimpleQueue() throws TimedOutException { + testSimpleConsumer(); + } + + @Test + public void verifyQueueLevelConsumer() throws TimedOutException { + checkQueueLevelConsumer(); + } + + @Test + public void verifyGroupConsumer() throws TimedOutException { + checkGroupConsumer(); + } +} diff --git a/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/tests/integration/RqueueRestControllerTest.java b/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/tests/integration/RqueueRestControllerTest.java index d828f063..1b75a80a 100644 --- a/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/tests/integration/RqueueRestControllerTest.java +++ b/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/tests/integration/RqueueRestControllerTest.java @@ -43,7 +43,7 @@ import com.github.sonus21.rqueue.spring.app.SpringApp.DeleteMessageListener; import com.github.sonus21.rqueue.test.dto.Email; import com.github.sonus21.rqueue.test.dto.Job; -import com.github.sonus21.rqueue.test.tests.SpringWebTestBase; +import com.github.sonus21.rqueue.test.common.SpringWebTestBase; import com.github.sonus21.rqueue.utils.Constants; import com.github.sonus21.rqueue.utils.MessageUtils; import com.github.sonus21.rqueue.utils.TimeoutUtils; diff --git a/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/tests/integration/RqueueViewControllerTest.java b/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/tests/integration/RqueueViewControllerTest.java index 01811197..c7aafb9f 100644 --- a/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/tests/integration/RqueueViewControllerTest.java +++ b/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/tests/integration/RqueueViewControllerTest.java @@ -20,28 +20,22 @@ import static org.junit.Assert.assertNotNull; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; -import com.github.sonus21.rqueue.exception.TimedOutException; import com.github.sonus21.rqueue.models.db.TaskStatus; import com.github.sonus21.rqueue.models.enums.AggregationType; import com.github.sonus21.rqueue.models.enums.NavTab; import com.github.sonus21.rqueue.spring.app.SpringApp; -import com.github.sonus21.rqueue.test.tests.SpringTestBase; +import com.github.sonus21.rqueue.test.common.SpringWebTestBase; import com.github.sonus21.test.RqueueSpringTestRunner; import java.util.Arrays; import lombok.extern.slf4j.Slf4j; import org.jtwig.spring.JtwigView; -import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.web.WebAppConfiguration; -import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.MvcResult; -import org.springframework.test.web.servlet.setup.MockMvcBuilders; import org.springframework.ui.ModelMap; -import org.springframework.web.context.WebApplicationContext; @ContextConfiguration(classes = SpringApp.class) @RunWith(RqueueSpringTestRunner.class) @@ -55,20 +49,11 @@ "notification.queue.active=false", "rqueue.web.statistic.history.day=180", }) -public class RqueueViewControllerTest extends SpringTestBase { - @Autowired private WebApplicationContext wac; - - private MockMvc mockMvc; - - @Before - public void init() throws TimedOutException { - this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).build(); - } - +public class RqueueViewControllerTest extends SpringWebTestBase { public void verifyBasicData(ModelMap model, NavTab navTab) { assertNotNull(model.get("latestVersion")); assertNotNull(model.get("version")); - assertNotNull(model.get("mavenRepoLink")); + assertNotNull(model.get("releaseLink")); assertNotNull(model.get("time")); assertNotNull(model.get("timeInMilli")); for (NavTab tab : NavTab.values()) { diff --git a/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/tests/integration/RqueueViewsDisabledTest.java b/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/tests/integration/RqueueViewsDisabledTest.java index 7964c6c7..02dcafee 100644 --- a/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/tests/integration/RqueueViewsDisabledTest.java +++ b/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/tests/integration/RqueueViewsDisabledTest.java @@ -31,7 +31,7 @@ import com.github.sonus21.rqueue.models.request.MessageMoveRequest; import com.github.sonus21.rqueue.spring.app.SpringApp; import com.github.sonus21.rqueue.test.dto.Job; -import com.github.sonus21.rqueue.test.tests.SpringWebTestBase; +import com.github.sonus21.rqueue.test.common.SpringWebTestBase; import com.github.sonus21.test.RqueueSpringTestRunner; import javax.servlet.http.HttpServletResponse; import lombok.extern.slf4j.Slf4j; diff --git a/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/tests/integration/SpringAppTest.java b/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/tests/integration/SpringAppTest.java index a6f8691c..8c4162bd 100644 --- a/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/tests/integration/SpringAppTest.java +++ b/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/tests/integration/SpringAppTest.java @@ -22,14 +22,19 @@ import com.github.sonus21.rqueue.core.QueueRegistry; import com.github.sonus21.rqueue.core.RqueueMessage; +import com.github.sonus21.rqueue.exception.QueueDoesNotExist; import com.github.sonus21.rqueue.exception.TimedOutException; import com.github.sonus21.rqueue.listener.QueueDetail; import com.github.sonus21.rqueue.spring.app.SpringApp; import com.github.sonus21.rqueue.test.dto.Email; +import com.github.sonus21.rqueue.test.dto.Sms; import com.github.sonus21.rqueue.test.tests.AllQueueMode; import com.github.sonus21.rqueue.utils.MessageUtils; import com.github.sonus21.rqueue.utils.TimeoutUtils; import com.github.sonus21.test.RqueueSpringTestRunner; +import java.time.Instant; +import java.util.Arrays; +import java.util.Date; import java.util.List; import java.util.Map; import lombok.extern.slf4j.Slf4j; @@ -90,7 +95,7 @@ public void verifyGroupConsumer() throws TimedOutException { } @Test - public void verifyEmailIsDiscardedAfter3RetriesDueToDlq() throws TimedOutException { + public void verifyDefaultDeadLetterQueueRetry() throws TimedOutException { Email email = Email.newInstance(); failureManager.createFailureDetail(email.getId(), 3, 10); rqueueMessageSender.enqueue(emailQueue, email); @@ -103,4 +108,35 @@ public void verifyEmailIsDiscardedAfter3RetriesDueToDlq() throws TimedOutExcepti messages.get(0), rqueueMessageSender.getMessageConverter()); assertEquals(email.getId(), email1.getId()); } + + @Test(expected = QueueDoesNotExist.class) + public void testQueueDoesNotExist() { + assertTrue(rqueueMessageSender.enqueue("job-push", Email.newInstance())); + } + + @Test + public void testOnlyPushMode() { + Date date = Date.from(Instant.now().plusMillis(1000)); + rqueueMessageSender.registerQueue("job-push"); + rqueueMessageSender.registerQueue("sms-push", "critical", "high", "low"); + + assertTrue(rqueueMessageSender.enqueue("job-push", Email.newInstance())); + assertTrue(rqueueMessageSender.enqueueAt("job-push", Email.newInstance(), date)); + assertTrue(rqueueMessageSender.enqueue("sms-push", Sms.newInstance())); + assertTrue(rqueueMessageSender.enqueueAt("sms-push", Sms.newInstance(), date.toInstant())); + assertTrue(rqueueMessageSender.enqueueWithPriority("sms-push", "critical", Sms.newInstance())); + assertTrue( + rqueueMessageSender.enqueueAtWithPriority("sms-push", "critical", Sms.newInstance(), date)); + assertTrue( + rqueueMessageSender.enqueueAtWithPriority( + "sms-push", "high", Sms.newInstance(), date.toInstant())); + assertTrue( + rqueueMessageSender.enqueueAtWithPriority( + "sms-push", "low", Sms.newInstance(), date.toInstant().toEpochMilli())); + assertEquals( + 8, + getMessageCount( + Arrays.asList( + "job-push", "sms-push", "sms-push_critical", "sms-push_high", "sms-push_low"))); + } } diff --git a/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/tests/integration/SpringMetricTest.java b/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/tests/integration/SpringMetricTest.java index 43a2b4b2..b77db738 100644 --- a/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/tests/integration/SpringMetricTest.java +++ b/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/tests/integration/SpringMetricTest.java @@ -18,7 +18,7 @@ import com.github.sonus21.rqueue.exception.TimedOutException; import com.github.sonus21.rqueue.spring.app.SpringApp; -import com.github.sonus21.rqueue.test.tests.MetricTestBase; +import com.github.sonus21.rqueue.test.tests.MetricTest; import com.github.sonus21.test.RqueueSpringTestRunner; import lombok.extern.slf4j.Slf4j; import org.junit.Test; @@ -32,7 +32,7 @@ @Slf4j @WebAppConfiguration @TestPropertySource(properties = {"rqueue.retry.per.poll=20", "spring.redis.port=7005"}) -public class SpringMetricTest extends MetricTestBase { +public class SpringMetricTest extends MetricTest { @Test public void delayedQueueStatus() throws TimedOutException { this.verifyDelayedQueueStatus(); diff --git a/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/tests/integration/StrictMultiLevelQueueListener.java b/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/tests/integration/StrictMultiLevelQueueListener.java index f454c0e5..158f9884 100644 --- a/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/tests/integration/StrictMultiLevelQueueListener.java +++ b/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/tests/integration/StrictMultiLevelQueueListener.java @@ -18,7 +18,7 @@ import com.github.sonus21.rqueue.exception.TimedOutException; import com.github.sonus21.rqueue.spring.app.SpringApp; -import com.github.sonus21.rqueue.test.tests.MultiLevelQueueListenerTestBase; +import com.github.sonus21.rqueue.test.tests.MultiLevelQueueTest; import com.github.sonus21.test.RqueueSpringTestRunner; import lombok.extern.slf4j.Slf4j; import org.junit.Test; @@ -43,7 +43,7 @@ "use.system.redis=false", "priority.mode=STRICT", }) -public class StrictMultiLevelQueueListener extends MultiLevelQueueListenerTestBase { +public class StrictMultiLevelQueueListener extends MultiLevelQueueTest { @Test public void simple() throws TimedOutException { diff --git a/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/tests/integration/StrictPriorityQueueListenerTest.java b/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/tests/integration/StrictPriorityQueueListenerTest.java index 6e6942a5..8d9ab806 100644 --- a/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/tests/integration/StrictPriorityQueueListenerTest.java +++ b/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/tests/integration/StrictPriorityQueueListenerTest.java @@ -18,7 +18,7 @@ import com.github.sonus21.rqueue.exception.TimedOutException; import com.github.sonus21.rqueue.spring.app.SpringApp; -import com.github.sonus21.rqueue.test.tests.GroupPriorityListener; +import com.github.sonus21.rqueue.test.tests.GroupPriorityTest; import com.github.sonus21.test.RqueueSpringTestRunner; import lombok.extern.slf4j.Slf4j; import org.junit.Test; @@ -45,7 +45,7 @@ "feed.generation.queue.active=true", "chat.indexing.queue.active=true" }) -public class StrictPriorityQueueListenerTest extends GroupPriorityListener { +public class StrictPriorityQueueListenerTest extends GroupPriorityTest { @Test public void simple() throws TimedOutException { checkGroupConsumer(); diff --git a/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/tests/integration/WeightedMultiLevelQueueListener.java b/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/tests/integration/WeightedMultiLevelQueueListener.java index 44fb9eef..0d7f0140 100644 --- a/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/tests/integration/WeightedMultiLevelQueueListener.java +++ b/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/tests/integration/WeightedMultiLevelQueueListener.java @@ -18,7 +18,7 @@ import com.github.sonus21.rqueue.exception.TimedOutException; import com.github.sonus21.rqueue.spring.app.SpringApp; -import com.github.sonus21.rqueue.test.tests.MultiLevelQueueListenerTestBase; +import com.github.sonus21.rqueue.test.tests.MultiLevelQueueTest; import com.github.sonus21.test.RqueueSpringTestRunner; import lombok.extern.slf4j.Slf4j; import org.junit.Test; @@ -42,10 +42,15 @@ "use.system.redis=false", "priority.mode=WEIGHTED", }) -public class WeightedMultiLevelQueueListener extends MultiLevelQueueListenerTestBase { +public class WeightedMultiLevelQueueListener extends MultiLevelQueueTest { @Test public void simple() throws TimedOutException { checkQueueLevelConsumer(); } + + @Test + public void delayed() throws TimedOutException { + checkQueueLevelConsumerWithDelay(); + } } diff --git a/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/tests/integration/WeightedPriorityQueueListener.java b/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/tests/integration/WeightedPriorityQueueListener.java index 7d930fe5..82142afd 100644 --- a/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/tests/integration/WeightedPriorityQueueListener.java +++ b/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/tests/integration/WeightedPriorityQueueListener.java @@ -18,7 +18,7 @@ import com.github.sonus21.rqueue.exception.TimedOutException; import com.github.sonus21.rqueue.spring.app.SpringApp; -import com.github.sonus21.rqueue.test.tests.GroupPriorityListener; +import com.github.sonus21.rqueue.test.tests.GroupPriorityTest; import com.github.sonus21.test.RqueueSpringTestRunner; import lombok.extern.slf4j.Slf4j; import org.junit.Test; @@ -45,7 +45,7 @@ "feed.generation.queue.active=true", "chat.indexing.queue.active=true" }) -public class WeightedPriorityQueueListener extends GroupPriorityListener { +public class WeightedPriorityQueueListener extends GroupPriorityTest { @Test public void simple() throws TimedOutException { checkGroupConsumer(); diff --git a/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/tests/unit/RqueueMessageConfigTest.java b/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/tests/unit/RqueueMessageConfigTest.java index f647164c..42fb9bc9 100644 --- a/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/tests/unit/RqueueMessageConfigTest.java +++ b/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/tests/unit/RqueueMessageConfigTest.java @@ -23,9 +23,9 @@ import com.github.sonus21.rqueue.config.SimpleRqueueListenerContainerFactory; import com.github.sonus21.rqueue.converter.GenericMessageConverter; +import com.github.sonus21.rqueue.core.RqueueMessageSender; import com.github.sonus21.rqueue.core.RqueueMessageTemplate; import com.github.sonus21.rqueue.listener.RqueueMessageHandler; -import com.github.sonus21.rqueue.core.RqueueMessageSender; import com.github.sonus21.rqueue.spring.RqueueListenerConfig; import java.util.ArrayList; import java.util.Collections; @@ -72,16 +72,6 @@ public void rqueueMessageHandlerReused() throws IllegalAccessException { assertEquals(rqueueMessageHandler.hashCode(), messageConfig.rqueueMessageHandler().hashCode()); } - @Test - public void rqueueMessageHandlerCreatedWithMessageConverters() throws IllegalAccessException { - SimpleRqueueListenerContainerFactory factory = new SimpleRqueueListenerContainerFactory(); - factory.setMessageConverters(messageConverterList); - RqueueListenerConfig messageConfig = new RqueueListenerConfig(); - FieldUtils.writeField(messageConfig, "simpleRqueueListenerContainerFactory", factory, true); - RqueueMessageHandler messageHandler = messageConfig.rqueueMessageHandler(); - assertEquals(messageConverterList.get(0), messageHandler.getMessageConverters().get(0)); - } - @Test public void rqueueMessageListenerContainer() throws IllegalAccessException { SimpleRqueueListenerContainerFactory factory = new SimpleRqueueListenerContainerFactory(); diff --git a/rqueue/src/main/java/com/github/sonus21/rqueue/config/RqueueConfig.java b/rqueue/src/main/java/com/github/sonus21/rqueue/config/RqueueConfig.java index ac0c900f..e308926f 100644 --- a/rqueue/src/main/java/com/github/sonus21/rqueue/config/RqueueConfig.java +++ b/rqueue/src/main/java/com/github/sonus21/rqueue/config/RqueueConfig.java @@ -32,7 +32,7 @@ public class RqueueConfig { private final boolean sharedConnection; private final int dbVersion; - @Value("${rqueue.version:2.0.0-RELEASE}") + @Value("${rqueue.version:2.0.1}") private String version; @Value("${rqueue.key.prefix:__rq::}") @@ -59,10 +59,6 @@ public class RqueueConfig { @Value("${rqueue.queues.key.suffix:queues}") private String queuesKeySuffix; - public String getQueuesKey() { - return prefix + queuesKeySuffix; - } - @Value("${rqueue.lock.key.prefix:lock::}") private String lockKeyPrefix; @@ -81,6 +77,10 @@ public String getQueuesKey() { @Value("${rqueue.default.queue.with.queue.level.priority:-1}") private int defaultQueueWithQueueLevelPriority; + public String getQueuesKey() { + return prefix + queuesKeySuffix; + } + public String getQueueName(String queueName) { if (dbVersion >= 2) { return prefix + simpleQueuePrefix + getTaggedName(queueName); diff --git a/rqueue/src/main/java/com/github/sonus21/rqueue/config/RqueueListenerBaseConfig.java b/rqueue/src/main/java/com/github/sonus21/rqueue/config/RqueueListenerBaseConfig.java index 4f2dd515..d100d0ca 100644 --- a/rqueue/src/main/java/com/github/sonus21/rqueue/config/RqueueListenerBaseConfig.java +++ b/rqueue/src/main/java/com/github/sonus21/rqueue/config/RqueueListenerBaseConfig.java @@ -52,6 +52,7 @@ * be very high based on the use case. */ public abstract class RqueueListenerBaseConfig { + public static final int MAX_DB_VERSION = 2; @Autowired(required = false) protected final SimpleRqueueListenerContainerFactory simpleRqueueListenerContainerFactory = @@ -63,12 +64,15 @@ public abstract class RqueueListenerBaseConfig { * Database for different ops. * * @param beanFactory configurable bean factory + * @param versionKey Rqueue db version key + * @param dbVersion database version * @return {@link RedisConnectionFactory} object. */ @Bean public RqueueConfig rqueueConfig( ConfigurableBeanFactory beanFactory, - @Value("${rqueue.version.key:__rq::version}") String versionKey) { + @Value("${rqueue.version.key:__rq::version}") String versionKey, + @Value("${rqueue.db.version:}") Integer dbVersion) { boolean sharedConnection = false; if (simpleRqueueListenerContainerFactory.getRedisConnectionFactory() == null) { sharedConnection = true; @@ -77,7 +81,15 @@ public RqueueConfig rqueueConfig( } RedisConnectionFactory connectionFactory = simpleRqueueListenerContainerFactory.getRedisConnectionFactory(); - int version = RedisUtils.updateAndGetVersion(connectionFactory, versionKey, 2); + int version; + if (dbVersion == null) { + version = RedisUtils.updateAndGetVersion(connectionFactory, versionKey, 2); + } else if (dbVersion >= 1 && dbVersion <= MAX_DB_VERSION) { + RedisUtils.setVersion(connectionFactory, versionKey, dbVersion); + version = dbVersion; + } else { + throw new IllegalStateException("Rqueue db version '" + dbVersion + "' is not correct"); + } return new RqueueConfig(connectionFactory, sharedConnection, version); } diff --git a/rqueue/src/main/java/com/github/sonus21/rqueue/config/SimpleRqueueListenerContainerFactory.java b/rqueue/src/main/java/com/github/sonus21/rqueue/config/SimpleRqueueListenerContainerFactory.java index 0355afae..44551a68 100644 --- a/rqueue/src/main/java/com/github/sonus21/rqueue/config/SimpleRqueueListenerContainerFactory.java +++ b/rqueue/src/main/java/com/github/sonus21/rqueue/config/SimpleRqueueListenerContainerFactory.java @@ -102,7 +102,7 @@ public AsyncTaskExecutor getTaskExecutor() { * the handler methods. If no {@link TaskExecutor} is set, a default one is created. * * @param taskExecutor The {@link TaskExecutor} used by the container. - * @see RqueueMessageListenerContainer#createDefaultTaskExecutor() + * @see RqueueMessageListenerContainer#createDefaultTaskExecutor(List) */ public void setTaskExecutor(AsyncTaskExecutor taskExecutor) { notNull(taskExecutor, "taskExecutor can not be null"); diff --git a/rqueue/src/main/java/com/github/sonus21/rqueue/converter/GenericMessageConverter.java b/rqueue/src/main/java/com/github/sonus21/rqueue/converter/GenericMessageConverter.java index fbb1bc17..04b97247 100644 --- a/rqueue/src/main/java/com/github/sonus21/rqueue/converter/GenericMessageConverter.java +++ b/rqueue/src/main/java/com/github/sonus21/rqueue/converter/GenericMessageConverter.java @@ -55,7 +55,7 @@ public Object fromMessage(Message message, Class targetClass) { String payload = (String) message.getPayload(); if (SerializationUtils.isJson(payload)) { Msg msg = objectMapper.readValue(payload, Msg.class); - Class c = Class.forName(msg.getName()); + Class c = Thread.currentThread().getContextClassLoader().loadClass(msg.getName()); return objectMapper.readValue(msg.msg, c); } } catch (IOException | ClassCastException | ClassNotFoundException e) { diff --git a/rqueue/src/main/java/com/github/sonus21/rqueue/core/QueueRegistry.java b/rqueue/src/main/java/com/github/sonus21/rqueue/core/QueueRegistry.java index a3e7ff83..e662feef 100644 --- a/rqueue/src/main/java/com/github/sonus21/rqueue/core/QueueRegistry.java +++ b/rqueue/src/main/java/com/github/sonus21/rqueue/core/QueueRegistry.java @@ -26,10 +26,10 @@ import java.util.stream.Collectors; public class QueueRegistry { - QueueRegistry() {} - - private static Map queueNameToDetail = new HashMap<>(); private static final Object lock = new Object(); + private static Map queueNameToDetail = new HashMap<>(); + + QueueRegistry() {} public static QueueDetail get(String queueName) { QueueDetail queueDetail = queueNameToDetail.get(queueName); diff --git a/rqueue/src/main/java/com/github/sonus21/rqueue/core/RedisScriptFactory.java b/rqueue/src/main/java/com/github/sonus21/rqueue/core/RedisScriptFactory.java index 06771212..a80ba727 100644 --- a/rqueue/src/main/java/com/github/sonus21/rqueue/core/RedisScriptFactory.java +++ b/rqueue/src/main/java/com/github/sonus21/rqueue/core/RedisScriptFactory.java @@ -32,7 +32,7 @@ static RedisScript getScript(ScriptType type) { script.setLocation(resource); switch (type) { case ADD_MESSAGE: - case REPLACE_MESSAGE: + case MOVE_MESSAGE: case PUSH_MESSAGE: case MOVE_MESSAGE_LIST_TO_LIST: case MOVE_MESSAGE_LIST_TO_ZSET: @@ -51,7 +51,7 @@ static RedisScript getScript(ScriptType type) { enum ScriptType { ADD_MESSAGE("scripts/add_message.lua"), POP_MESSAGE("scripts/pop_message.lua"), - REPLACE_MESSAGE("scripts/replace_message.lua"), + MOVE_MESSAGE("scripts/move_message.lua"), PUSH_MESSAGE("scripts/push_message.lua"), MOVE_MESSAGE_LIST_TO_LIST("scripts/move_message_list_to_list.lua"), MOVE_MESSAGE_LIST_TO_ZSET("scripts/move_message_list_to_zset.lua"), diff --git a/rqueue/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageSender.java b/rqueue/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageSender.java index 990467ee..f1e84e45 100644 --- a/rqueue/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageSender.java +++ b/rqueue/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageSender.java @@ -18,7 +18,11 @@ import com.github.sonus21.rqueue.annotation.RqueueListener; import com.github.sonus21.rqueue.utils.Constants; +import java.time.Duration; +import java.time.Instant; +import java.util.Date; import java.util.List; +import java.util.concurrent.TimeUnit; import org.springframework.messaging.converter.MessageConverter; /** @@ -26,66 +30,165 @@ * from this interface to send message over any queue. Queue must exist, if a queue does not exist * then it will throw an error of the {@link com.github.sonus21.rqueue.exception.QueueDoesNotExist}. * - *

There're two types of interfaces in this 1. enqueueXYZ 2. enqueueInXYZ + *

There're three types of interfaces in this 1. enqueueXYZ 2. enqueueInXYZ 3. enqueueAtXYZ * - *

enqueueXYZ method is used to send the message without any delay, while if any message send - * using enqueueInXYZ would be delayed for the specified period. The last parameter of enqueueIn is - * the delay from the current time. For example if you want to send a message and that should be - * consumed in 10 seconds then use one of the enqueueIn with delay = 10000, delay in these methods - * are in milli seconds. + *

Messages send using enqueueXYZ shall be consume as soon as possible + * + *

Messages send using enqueueInXYZ shall be consumed once the given time is elapsed, like in 30 + * seconds. + * + *

Messages send using enqueueAtXYZ shall be consumed as soon as the given time is reached for + * example 3PM tomorrow. * * @author Sonu Kumar */ public interface RqueueMessageSender { /** - * Submit a message on given queue without any delay, listener would try to consume this message - * immediately but due to heavy load message consumption can be delayed if message producer rate - * is higher than the rate at consumer consume the messages. + * Enqueue a message on given queue without any delay, consume as soon as possible. * + * @deprecated migrate to {@link #enqueue(String, Object)} * @param queueName on which queue message has to be send * @param message message object it could be any arbitrary object. * @return message was submitted successfully or failed. */ @Deprecated - boolean put(String queueName, Object message); + default boolean put(String queueName, Object message) { + return enqueue(queueName, message); + } + /** + * Enqueue a message on given queue without any delay, consume as soon as possible. + * + * @param queueName on which queue message has to be send + * @param message message object it could be any arbitrary object. + * @return message was submitted successfully or failed. + */ boolean enqueue(String queueName, Object message); /** - * This is the extension to the method {@link #enqueue(String, Object)}, in this we can specify - * when this message would be visible to the consumer. + * Enqueue a message on the given queue with the provided delay. It will be available to consume + * as soon as the delay elapse. * * @param queueName on which queue message has to be send * @param message message object it could be any arbitrary object. - * @param delayInMilliSecs delay in milli seconds, this message would be only visible to the - * listener when number of millisecond has elapsed. + * @param delayInMilliSecs delay in milli seconds * @return message was submitted successfully or failed. */ @Deprecated - boolean put(String queueName, Object message, long delayInMilliSecs); + default boolean put(String queueName, Object message, long delayInMilliSecs) { + return enqueueIn(queueName, message, delayInMilliSecs); + } + /** + * Schedule a message on the given queue with the provided delay. It will be available to consume + * as soon as the delay elapse. + * + * @param queueName on which queue message has to be send + * @param message message object it could be any arbitrary object. + * @param delayInMilliSecs delay in milli seconds + * @return message was submitted successfully or failed. + */ boolean enqueueIn(String queueName, Object message, long delayInMilliSecs); /** - * This is an extension to the method {@link #enqueue(String, Object)}. By default container would - * try to deliver the same message for {@link Integer#MAX_VALUE} times, but that can be either - * overridden using {@link RqueueListener#numRetries()}, even that value can be overridden using - * this method. + * Schedule a message on the given queue with the provided delay. It will be available to consume + * as soon as the delay elapse. * * @param queueName on which queue message has to be send * @param message message object it could be any arbitrary object. - * @param retryCount how many times a message would be retried, before it can be discarded or sent + * @param delay time to wait before it can be executed. + * @return message was submitted successfully or failed. + */ + default boolean enqueueIn(String queueName, Object message, Duration delay) { + return enqueueIn(queueName, message, delay.toMillis()); + } + + /** + * Schedule a message on the given queue with the provided delay. It will be available to consume + * as soon as the specified delay elapse. + * + * @param queueName on which queue message has to be send + * @param message message object it could be any arbitrary object. + * @param delay time to wait before it can be executed. + * @param unit unit of the delay + * @return message was submitted successfully or failed. + */ + default boolean enqueueIn(String queueName, Object message, long delay, TimeUnit unit) { + return enqueueIn(queueName, message, unit.toMillis(delay)); + } + + /** + * Schedule a message on the given queue at the provided time. It will be available to consume as + * soon as the given time is reached. + * + * @param queueName on which queue message has to be send + * @param message message object it could be any arbitrary object. + * @param startTimeInMilliSeconds time at which this message has to be consumed. + * @return message was submitted successfully or failed. + */ + default boolean enqueueAt(String queueName, Object message, long startTimeInMilliSeconds) { + return enqueueIn(queueName, message, startTimeInMilliSeconds - System.currentTimeMillis()); + } + + /** + * Schedule a message on the given queue at the provided time. It will be available to consume as + * soon as the given time is reached. + * + * @param queueName on which queue message has to be send + * @param message message object it could be any arbitrary object. + * @param starTime time at which this message has to be consumed. + * @return message was submitted successfully or failed. + */ + default boolean enqueueAt(String queueName, Object message, Instant starTime) { + return enqueueAt(queueName, message, starTime.toEpochMilli()); + } + + /** + * Schedule a message on the given queue at the provided time. It will be available to consume as + * soon as the given time is reached. + * + * @param queueName on which queue message has to be send + * @param message message object it could be any arbitrary object. + * @param starTime time at which this message has to be consumed. + * @return message was submitted successfully or failed. + */ + default boolean enqueueAt(String queueName, Object message, Date starTime) { + return enqueueAt(queueName, message, starTime.toInstant()); + } + + /** + * Enqueue a message on the given queue with the given retry count. This message would not be + * consumed more than the specified time due to failure in underlying systems. + * + * @deprecated migrate to {@link #enqueueWithPriority(String, String, Object)} + * @param queueName on which queue message has to be send + * @param message message object it could be any arbitrary object. + * @param retryCount how many times a message would be retried, before it can be discarded or send * to dead letter queue configured using {@link RqueueListener#numRetries()} * @return message was submitted successfully or failed. */ @Deprecated - boolean put(String queueName, Object message, int retryCount); + default boolean put(String queueName, Object message, int retryCount) { + return enqueueWithRetry(queueName, message, retryCount); + } + /** + * Enqueue a message on the given queue with the given retry count. This message would not be + * consumed more than the specified time due to failure in underlying systems. + * + * @param queueName on which queue message has to be send + * @param message message object it could be any arbitrary object. + * @param retryCount how many times a message would be retried, before it can be discarded or send + * to dead letter queue configured using {@link RqueueListener#numRetries()} + * @return message was submitted successfully or failed. + */ boolean enqueueWithRetry(String queueName, Object message, int retryCount); /** - * Enqueue a task that would be scheduled to run after N milli seconds. + * Enqueue a message on the given queue that would be scheduled to run in the specified milli + * seconds. * + * @deprecated migrate to {@link #enqueueInWithRetry(String, Object, int, long)} * @param queueName on which queue message has to be send * @param message message object it could be any arbitrary object. * @param retryCount how many times a message would be retried, before it can be discarded or sent @@ -95,15 +198,26 @@ public interface RqueueMessageSender { * @return message was submitted successfully or failed. */ @Deprecated - boolean put(String queueName, Object message, int retryCount, long delayInMilliSecs); + default boolean put(String queueName, Object message, int retryCount, long delayInMilliSecs) { + return enqueueInWithRetry(queueName, message, retryCount, delayInMilliSecs); + } + /** + * Enqueue a task that would be scheduled to run in the specified milli seconds. + * + * @param queueName on which queue message has to be send + * @param message message object it could be any arbitrary object. + * @param retryCount how many times a message would be retried, before it can be discarded or sent + * to dead letter queue configured using {@link RqueueListener#numRetries()} ()} + * @param delayInMilliSecs delay in milli seconds, this message would be only visible to the + * listener when number of millisecond has elapsed. + * @return message was submitted successfully or failed. + */ boolean enqueueInWithRetry( String queueName, Object message, int retryCount, long delayInMilliSecs); /** - * Submit a message on given queue without any delay, listener would try to consume this message - * immediately but due to heavy load message consumption can be delayed if message producer rate - * is higher than the rate at consumer consume the messages. + * Enqueue a message on given queue, that will be consumed as soon as possible. * * @param queueName on which queue message has to be send * @param priority the priority name for this message @@ -113,18 +227,95 @@ boolean enqueueInWithRetry( boolean enqueueWithPriority(String queueName, String priority, Object message); /** - * Submit a task to the given queue at the given priority level. + * Schedule a message on the given queue at the provided time. It will be executed as soon as the + * given delay is elapse. * * @param queueName on which queue message has to be send * @param priority the name of the priority level * @param message message object it could be any arbitrary object. - * @param delayInMilliSecs delay in milli seconds, this message would be only visible to the - * listener when number of millisecond has elapsed. + * @param delayInMilliSecs delay in milli seconds * @return message was submitted successfully or failed. */ boolean enqueueInWithPriority( String queueName, String priority, Object message, long delayInMilliSecs); + /** + * Schedule a message on the given queue at the provided time. It will be executed as soon as the + * given delay is elapse. + * + * @param queueName on which queue message has to be send + * @param priority the name of the priority level + * @param message message object it could be any arbitrary object. + * @param delay time to wait before it can be consumed. + * @return message was submitted successfully or failed. + */ + default boolean enqueueInWithPriority( + String queueName, String priority, Object message, Duration delay) { + return enqueueInWithPriority(queueName, priority, message, delay.toMillis()); + } + + /** + * Schedule a message on the given queue at the provided time. It will be executed as soon as the + * given delay is elapse. + * + * @param queueName on which queue message has to be send + * @param priority the name of the priority level + * @param message message object it could be any arbitrary object. + * @param delay time to wait before it can be consumed. + * @param unit unit of the delay + * @return message was submitted successfully or failed. + */ + default boolean enqueueInWithPriority( + String queueName, String priority, Object message, long delay, TimeUnit unit) { + return enqueueInWithPriority(queueName, priority, message, unit.toMillis(delay)); + } + + /** + * Schedule a message on the given queue at the provided time. It will be executed as soon as the + * given time is reached, time must be in the future. + * + * @param queueName on which queue message has to be send + * @param priority the name of the priority level + * @param message message object it could be any arbitrary object. + * @param startTimeInMilliSecond time at which the message would be consumed. + * @return message was submitted successfully or failed. + */ + default boolean enqueueAtWithPriority( + String queueName, String priority, Object message, long startTimeInMilliSecond) { + return enqueueInWithPriority( + queueName, priority, message, startTimeInMilliSecond - System.currentTimeMillis()); + } + + /** + * Schedule a message on the given queue at the provided time. It will be executed as soon as the + * given time is reached, time must be in the future. + * + * @param queueName on which queue message has to be send + * @param priority the name of the priority level + * @param message message object it could be any arbitrary object. + * @param startTime time at which message is supposed to consume + * @return message was submitted successfully or failed. + */ + default boolean enqueueAtWithPriority( + String queueName, String priority, Object message, Instant startTime) { + return enqueueAtWithPriority(queueName, priority, message, startTime.toEpochMilli()); + } + + /** + * Schedule a message on the given queue at the provided time. It will be executed as soon as the + * given time is reached, time must be in the future. + * + * @param queueName on which queue message has to be send + * @param priority the name of the priority level + * @param message message object it could be any arbitrary object. + * @param startTime time at which message would be consumed. + * @return message was submitted successfully or failed. + */ + default boolean enqueueAtWithPriority( + String queueName, String priority, Object message, Date startTime) { + return enqueueAtWithPriority(queueName, priority, message, startTime.toInstant()); + } + /** * Move messages from Dead Letter queue to the destination queue. This push the messages at the * FRONT of destination queue, so that it can be reprocessed as soon as possible. @@ -171,5 +362,18 @@ boolean moveMessageFromDeadLetterToQueue( */ List getAllMessages(String queueName); + /** + * Get all registered message converters. + * + * @return list of message converters. + */ MessageConverter getMessageConverter(); + + /** + * Use this method to register any queue, that's only used for sending message. + * + * @param name name of the queue + * @param priorities list of priorities to be used while sending message on this queue. + */ + void registerQueue(String name, String... priorities); } diff --git a/rqueue/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageSenderImpl.java b/rqueue/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageSenderImpl.java index 151ae584..07966923 100644 --- a/rqueue/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageSenderImpl.java +++ b/rqueue/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageSenderImpl.java @@ -28,6 +28,7 @@ import static org.springframework.util.Assert.notNull; import com.github.sonus21.rqueue.common.RqueueRedisTemplate; +import com.github.sonus21.rqueue.config.RqueueConfig; import com.github.sonus21.rqueue.converter.GenericMessageConverter; import com.github.sonus21.rqueue.listener.QueueDetail; import com.github.sonus21.rqueue.models.MessageMoveResult; @@ -45,9 +46,10 @@ @Slf4j public class RqueueMessageSenderImpl implements RqueueMessageSender { - private RqueueMessageTemplate messageTemplate; private final CompositeMessageConverter messageConverter; + private RqueueMessageTemplate messageTemplate; @Autowired private RqueueRedisTemplate stringRqueueRedisTemplate; + @Autowired private RqueueConfig rqueueConfig; private RqueueMessageSenderImpl( RqueueMessageTemplate messageTemplate, @@ -69,26 +71,6 @@ public RqueueMessageSenderImpl( this(messageTemplate, messageConverters, false); } - @Override - public boolean put(String queueName, Object message) { - return enqueue(queueName, message); - } - - @Override - public boolean put(String queueName, Object message, long delayInMilliSecs) { - return enqueueIn(queueName, message, delayInMilliSecs); - } - - @Override - public boolean put(String queueName, Object message, int retryCount) { - return enqueueWithRetry(queueName, message, retryCount); - } - - @Override - public boolean put(String queueName, Object message, int retryCount, long delayInMilliSecs) { - return enqueueInWithRetry(queueName, message, retryCount, delayInMilliSecs); - } - @Override public boolean enqueue(String queueName, Object message) { validateQueue(queueName); @@ -220,7 +202,7 @@ private boolean pushMessage( rqueueMessage); } } catch (Exception e) { - log.error("Message could not be pushed ", e); + log.error("Queue: {} Message {} could not be pushed {}", queueName, rqueueMessage, e); return false; } return true; @@ -238,4 +220,36 @@ private List getMessageConverters( messageConverterList.addAll(messageConverters); return messageConverterList; } + + @Override + public void registerQueue(String queueName, String... priorities) { + validateQueue(queueName); + notNull(priorities, "priorities cannot be null"); + QueueDetail queueDetail = + QueueDetail.builder() + .name(queueName) + .active(false) + .queueName(rqueueConfig.getQueueName(queueName)) + .delayedQueueName(rqueueConfig.getDelayedQueueName(queueName)) + .delayedQueueChannelName(rqueueConfig.getDelayedQueueChannelName(queueName)) + .processingQueueName(rqueueConfig.getProcessingQueueName(queueName)) + .processingQueueChannelName(rqueueConfig.getProcessingQueueChannelName(queueName)) + .build(); + QueueRegistry.register(queueDetail); + for (String priority : priorities) { + String suffix = PriorityUtils.getSuffix(priority); + queueDetail = + QueueDetail.builder() + .name(queueName + suffix) + .active(false) + .queueName(rqueueConfig.getQueueName(queueName) + suffix) + .delayedQueueName(rqueueConfig.getDelayedQueueName(queueName) + suffix) + .delayedQueueChannelName(rqueueConfig.getDelayedQueueChannelName(queueName) + suffix) + .processingQueueName(rqueueConfig.getProcessingQueueName(queueName) + suffix) + .processingQueueChannelName( + rqueueConfig.getProcessingQueueChannelName(queueName) + suffix) + .build(); + QueueRegistry.register(queueDetail); + } + } } diff --git a/rqueue/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageTemplateImpl.java b/rqueue/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageTemplateImpl.java index 6d45a6c9..7fdb0268 100644 --- a/rqueue/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageTemplateImpl.java +++ b/rqueue/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageTemplateImpl.java @@ -26,6 +26,7 @@ import java.util.Arrays; import java.util.List; import java.util.Set; +import lombok.extern.slf4j.Slf4j; import org.springframework.data.redis.connection.RedisConnectionFactory; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.data.redis.core.ZSetOperations.TypedTuple; @@ -39,6 +40,7 @@ *

It communicates with the Redis using Lua script and direct calls. */ @SuppressWarnings("unchecked") +@Slf4j public class RqueueMessageTemplateImpl extends RqueueRedisTemplate implements RqueueMessageTemplate { private DefaultScriptExecutor scriptExecutor; @@ -90,13 +92,17 @@ public Boolean addToZset(String zsetName, RqueueMessage rqueueMessage, long scor @Override public void moveMessage( String srcZsetName, String tgtZsetName, RqueueMessage src, RqueueMessage tgt, long delay) { - RedisScript script = (RedisScript) getScript(ScriptType.REPLACE_MESSAGE); - scriptExecutor.execute( - script, - Arrays.asList(srcZsetName, tgtZsetName), - src, - tgt, - System.currentTimeMillis() + delay); + RedisScript script = (RedisScript) getScript(ScriptType.MOVE_MESSAGE); + Long response = + scriptExecutor.execute( + script, + Arrays.asList(srcZsetName, tgtZsetName), + src, + tgt, + System.currentTimeMillis() + delay); + if (response == null) { + log.error("Duplicate processing for the message {}", src); + } } @Override diff --git a/rqueue/src/main/java/com/github/sonus21/rqueue/core/support/MessageProcessor.java b/rqueue/src/main/java/com/github/sonus21/rqueue/core/support/MessageProcessor.java index f3bac9a5..438e2dcd 100644 --- a/rqueue/src/main/java/com/github/sonus21/rqueue/core/support/MessageProcessor.java +++ b/rqueue/src/main/java/com/github/sonus21/rqueue/core/support/MessageProcessor.java @@ -16,6 +16,8 @@ package com.github.sonus21.rqueue.core.support; +import com.github.sonus21.rqueue.core.RqueueMessage; + /** * This interface can be used to take some action when ever a message is processed * @@ -39,7 +41,22 @@ public interface MessageProcessor { * @param message message * @return true/false. */ + @Deprecated default boolean process(Object message) { return true; } + + /** + * This method would be called with the specified object, this will happen only when message is + * deserialize successfully. Return value is used for pre-processing, where if caller returns true + * then only message would be executed otherwise it will be discarded, returning false means task + * to ignore. It's considered that message has to be deleted. + * + * @param message message + * @param rqueueMessage rqueue message object + * @return true/false. + */ + default boolean process(Object message, RqueueMessage rqueueMessage) { + return process(message); + } } diff --git a/rqueue/src/main/java/com/github/sonus21/rqueue/listener/RqueueExecutor.java b/rqueue/src/main/java/com/github/sonus21/rqueue/listener/RqueueExecutor.java index fda87108..1f9313a1 100644 --- a/rqueue/src/main/java/com/github/sonus21/rqueue/listener/RqueueExecutor.java +++ b/rqueue/src/main/java/com/github/sonus21/rqueue/listener/RqueueExecutor.java @@ -18,10 +18,10 @@ import static com.github.sonus21.rqueue.utils.Constants.DELTA_BETWEEN_RE_ENQUEUE_TIME; import static com.github.sonus21.rqueue.utils.Constants.SECONDS_IN_A_WEEK; -import static org.springframework.util.Assert.notNull; import com.github.sonus21.rqueue.core.RqueueMessage; import com.github.sonus21.rqueue.core.support.MessageProcessor; +import com.github.sonus21.rqueue.exception.UnknownSwitchCase; import com.github.sonus21.rqueue.metrics.RqueueCounter; import com.github.sonus21.rqueue.models.db.MessageMetadata; import com.github.sonus21.rqueue.models.db.TaskStatus; @@ -37,7 +37,8 @@ import lombok.extern.slf4j.Slf4j; import org.slf4j.event.Level; import org.springframework.messaging.Message; -import org.springframework.messaging.support.GenericMessage; +import org.springframework.messaging.MessagingException; +import org.springframework.messaging.support.MessageBuilder; @Slf4j class RqueueExecutor extends MessageContainerBase { @@ -70,11 +71,12 @@ class RqueueExecutor extends MessageContainerBase { this.retryPerPoll = retryPerPoll; this.taskExecutionBackoff = taskExecutionBackoff; this.message = - new GenericMessage<>( - rqueueMessage.getMessage(), MessageUtils.getMessageHeader(queueDetail.getName())); + MessageBuilder.createMessage( + rqueueMessage.getMessage(), + RqueueMessageHeaders.buildMessageHeaders(queueDetail.getName(), rqueueMessage)); try { this.userMessage = - MessageUtils.convertMessageToObject(message, rqueueMessageHandler.getMessageConverters()); + MessageUtils.convertMessageToObject(message, rqueueMessageHandler.getMessageConverter()); } catch (Exception e) { log(Level.ERROR, "Unable to convert message {}", e, rqueueMessage.getMessage()); } @@ -112,20 +114,19 @@ private void callMessageProcessor(TaskStatus status, RqueueMessage rqueueMessage if (messageProcessor != null) { try { log(Level.DEBUG, "Calling {} processor for {}", null, status, rqueueMessage); - messageProcessor.process(userMessage); + messageProcessor.process(userMessage, rqueueMessage); } catch (Exception e) { log(Level.ERROR, "Message processor {} call failed", e, status); } } } - @SuppressWarnings("ConstantConditions") - private void updateCounter(boolean failOrExecution) { - RqueueCounter rqueueCounter = container.get().getRqueueCounter(); + private void updateCounter(boolean fail) { + RqueueCounter rqueueCounter = Objects.requireNonNull(container.get()).getRqueueCounter(); if (rqueueCounter == null) { return; } - if (failOrExecution) { + if (fail) { rqueueCounter.updateFailureCount(queueDetail.getName()); } else { rqueueCounter.updateExecutionCount(queueDetail.getName()); @@ -141,7 +142,7 @@ private void publishEvent(TaskStatus status, long jobExecutionStartTime) { } } - private void addOrDeleteMetadata(long jobExecutionTime, boolean saveOrDelete) { + private void addOrDeleteMetadata(long jobExecutionStartTime, boolean saveOrDelete) { if (messageMetadata == null) { messageMetadata = rqueueMessageMetadataService.get(messageMetadataId); } @@ -149,11 +150,11 @@ private void addOrDeleteMetadata(long jobExecutionTime, boolean saveOrDelete) { messageMetadata = new MessageMetadata(messageMetadataId, rqueueMessage.getId()); // do not call db delete method if (!saveOrDelete) { - messageMetadata.addExecutionTime(jobExecutionTime); + messageMetadata.addExecutionTime(jobExecutionStartTime); return; } } - messageMetadata.addExecutionTime(jobExecutionTime); + messageMetadata.addExecutionTime(jobExecutionStartTime); if (saveOrDelete) { Objects.requireNonNull(container.get()) .getRqueueMessageMetadataService() @@ -163,28 +164,26 @@ private void addOrDeleteMetadata(long jobExecutionTime, boolean saveOrDelete) { } } - private void deleteMessage( - TaskStatus status, int currentFailureCount, long jobExecutionStartTime) { + private void deleteMessage(TaskStatus status, int failureCount, long jobExecutionStartTime) { getRqueueMessageTemplate() .removeElementFromZset(queueDetail.getProcessingQueueName(), rqueueMessage); - rqueueMessage.setFailureCount(currentFailureCount); + rqueueMessage.setFailureCount(failureCount); callMessageProcessor(status, rqueueMessage); publishEvent(status, jobExecutionStartTime); } - private void moveMessageToDlq(int currentFailureCount, long jobExecutionStartTime) + private void moveMessageToDlq(int failureCount, long jobExecutionStartTime) throws CloneNotSupportedException { if (isWarningEnabled()) { log( Level.WARN, - "Message {} Moved to dead letter queue: {}, dead letter queue: {}", + "Message {} Moved to dead letter queue: {}", null, userMessage, - queueDetail.getName(), queueDetail.getDeadLetterQueueName()); } RqueueMessage newMessage = rqueueMessage.clone(); - newMessage.setFailureCount(currentFailureCount); + newMessage.setFailureCount(failureCount); newMessage.updateReEnqueuedAt(); callMessageProcessor(TaskStatus.MOVED_TO_DLQ, newMessage); RedisUtils.executePipeLine( @@ -201,20 +200,13 @@ private void moveMessageToDlq(int currentFailureCount, long jobExecutionStartTim publishEvent(TaskStatus.MOVED_TO_DLQ, jobExecutionStartTime); } - private void parkMessageForRetry(int currentFailureCount, long jobExecutionStartTime, long delay) + private void parkMessageForRetry(int failureCount, long jobExecutionStartTime, long delay) throws CloneNotSupportedException { if (isDebugEnabled()) { - log( - Level.DEBUG, - "Message {} will be retried in {}Ms, queue: {}, Redis Queue: {}", - null, - userMessage, - delay, - queueDetail.getName(), - queueDetail.getQueueName()); + log(Level.DEBUG, "Message {} will be retried in {}Ms", null, userMessage, delay); } RqueueMessage newMessage = rqueueMessage.clone(); - newMessage.setFailureCount(currentFailureCount); + newMessage.setFailureCount(failureCount); newMessage.updateReEnqueuedAt(); getRqueueMessageTemplate() .moveMessage( @@ -226,98 +218,83 @@ private void parkMessageForRetry(int currentFailureCount, long jobExecutionStart addOrDeleteMetadata(jobExecutionStartTime, true); } - private void discardMessage(int currentFailureCount, long jobExecutionStartTime) { + private void discardMessage(int failureCount, long jobExecutionStartTime) { if (isDebugEnabled()) { - log( - Level.DEBUG, - "Message {} discarded due to retry limit exhaust queue: {}", - null, - userMessage, - queueDetail.getName()); + log(Level.DEBUG, "Message {} discarded due to retry limit exhaust", null, userMessage); } - deleteMessage(TaskStatus.DISCARDED, currentFailureCount, jobExecutionStartTime); + deleteMessage(TaskStatus.DISCARDED, failureCount, jobExecutionStartTime); } - private void handleManualDeletion(int currentFailureCount, long jobExecutionStartTime) { + private void handleManualDeletion(int failureCount, long jobExecutionStartTime) { if (isDebugEnabled()) { - log( - Level.DEBUG, - "Message Deleted manually {} successfully, Queue: {}", - null, - rqueueMessage, - queueDetail.getName()); + log(Level.DEBUG, "Message Deleted {} successfully", null, rqueueMessage); } - deleteMessage(TaskStatus.DELETED, currentFailureCount, jobExecutionStartTime); + deleteMessage(TaskStatus.DELETED, failureCount, jobExecutionStartTime); } - private void handleSuccessFullExecution(int currentFailureCount, long jobExecutionStartTime) { + private void handleSuccessFullExecution(int failureCount, long jobExecutionStartTime) { if (isDebugEnabled()) { - log( - Level.DEBUG, - "Message consumed {} successfully, Queue: {}", - null, - rqueueMessage, - queueDetail.getName()); + log(Level.DEBUG, "Message consumed {} successfully", null, rqueueMessage); } - deleteMessage(TaskStatus.SUCCESSFUL, currentFailureCount, jobExecutionStartTime); + deleteMessage(TaskStatus.SUCCESSFUL, failureCount, jobExecutionStartTime); } - private void handleLimitExceededMessage(int currentFailureCount, long jobExecutionStartTime) + private void handleRetryExceededMessage(int failureCount, long jobExecutionStartTime) throws CloneNotSupportedException { if (queueDetail.isDlqSet()) { - moveMessageToDlq(currentFailureCount, jobExecutionStartTime); + moveMessageToDlq(failureCount, jobExecutionStartTime); } else { - discardMessage(currentFailureCount, jobExecutionStartTime); + discardMessage(failureCount, jobExecutionStartTime); } } - private void handleFailure(int currentFailureCount, int maxRetryCount, long jobExecutionStartTime) + private void handleFailure(int failureCount, long jobExecutionStartTime) throws CloneNotSupportedException { - if (currentFailureCount < maxRetryCount) { - long delay = - taskExecutionBackoff.nextBackOff(userMessage, rqueueMessage, currentFailureCount); + int maxRetryCount = getMaxRetryCount(); + if (failureCount < maxRetryCount) { + long delay = taskExecutionBackoff.nextBackOff(userMessage, rqueueMessage, failureCount); if (delay == TaskExecutionBackOff.STOP) { - handleLimitExceededMessage(currentFailureCount, jobExecutionStartTime); + handleRetryExceededMessage(failureCount, jobExecutionStartTime); } else { - parkMessageForRetry(currentFailureCount, jobExecutionStartTime, delay); + parkMessageForRetry(failureCount, jobExecutionStartTime, delay); } } else { - handleLimitExceededMessage(currentFailureCount, jobExecutionStartTime); + handleRetryExceededMessage(failureCount, jobExecutionStartTime); } } private void handlePostProcessing( - boolean executed, - boolean deleted, - boolean ignored, - int currentFailureCount, - int maxRetryCount, - long jobExecutionStartTime) { - if (!isQueueActive(queueDetail.getName())) { + TaskStatus status, int failureCount, long jobExecutionStartTime) { + if (status == TaskStatus.QUEUE_INACTIVE) { return; } try { - if (ignored) { - handleIgnoredMessage(currentFailureCount, jobExecutionStartTime); - } else if (deleted) { - handleManualDeletion(currentFailureCount, jobExecutionStartTime); - } else { - if (!executed) { - handleFailure(currentFailureCount, maxRetryCount, jobExecutionStartTime); - } else { - handleSuccessFullExecution(currentFailureCount, jobExecutionStartTime); - } + switch (status) { + case SUCCESSFUL: + handleSuccessFullExecution(failureCount, jobExecutionStartTime); + break; + case DELETED: + handleManualDeletion(failureCount, jobExecutionStartTime); + break; + case IGNORED: + handleIgnoredMessage(failureCount, jobExecutionStartTime); + break; + case FAILED: + handleFailure(failureCount, jobExecutionStartTime); + break; + default: + throw new UnknownSwitchCase(String.valueOf(status)); } } catch (Exception e) { log(Level.ERROR, "Error occurred in post processing", e); } } - private void handleIgnoredMessage(int currentFailureCount, long jobExecutionStartTime) { + private void handleIgnoredMessage(int failureCount, long jobExecutionStartTime) { if (isDebugEnabled()) { log(Level.DEBUG, "Message {} ignored, Queue: {}", null, rqueueMessage, queueDetail.getName()); } - deleteMessage(TaskStatus.IGNORED, currentFailureCount, jobExecutionStartTime); + deleteMessage(TaskStatus.IGNORED, failureCount, jobExecutionStartTime); } private long getMaxProcessingTime() { @@ -326,8 +303,7 @@ private long getMaxProcessingTime() { - DELTA_BETWEEN_RE_ENQUEUE_TIME; } - private boolean isMessageDeleted(String id) { - notNull(id, "Message id must be present"); + private boolean isMessageDeleted() { messageMetadata = rqueueMessageMetadataService.get(messageMetadataId); if (messageMetadata == null) { return false; @@ -335,10 +311,10 @@ private boolean isMessageDeleted(String id) { return messageMetadata.isDeleted(); } - private boolean shouldProcess() { - return Objects.requireNonNull(container.get()) + private boolean shouldIgnore() { + return !Objects.requireNonNull(container.get()) .getPreExecutionMessageProcessor() - .process(userMessage); + .process(userMessage, rqueueMessage); } private int getRetryCount() { @@ -349,45 +325,51 @@ private int getRetryCount() { return Math.min(retryPerPoll, maxRetry); } + private boolean queueActive() { + return isQueueActive(queueDetail.getName()); + } + + private TaskStatus getStatus() { + if (!queueActive()) { + return TaskStatus.QUEUE_INACTIVE; + } + if (shouldIgnore()) { + return TaskStatus.IGNORED; + } + if (isMessageDeleted()) { + return TaskStatus.DELETED; + } + return null; + } + @Override void start() { - boolean executed = false; - int currentFailureCount = rqueueMessage.getFailureCount(); - int maxRetryCount = getMaxRetryCount(); - long maxRetryTime = getMaxProcessingTime(); + int failureCount = rqueueMessage.getFailureCount(); + long maxProcessingTime = getMaxProcessingTime(); long startTime = System.currentTimeMillis(); - boolean deleted = false; - boolean ignored = false; int retryCount = getRetryCount(); + TaskStatus status; try { do { - if (!isQueueActive(queueDetail.getName())) { - return; - } - if (!shouldProcess()) { - ignored = true; - } else if (isMessageDeleted(rqueueMessage.getId())) { - deleted = true; - } - if (ignored || deleted) { + status = getStatus(); + if (status != null) { break; } try { updateCounter(false); rqueueMessageHandler.handleMessage(message); - executed = true; + status = TaskStatus.SUCCESSFUL; + } catch (MessagingException e) { + updateCounter(true); + failureCount += 1; } catch (Exception e) { - log(Level.ERROR, "Message consumer failed", e); updateCounter(true); - currentFailureCount += 1; + failureCount += 1; + log(Level.ERROR, "Message execution failed", e); } retryCount--; - } while (currentFailureCount < maxRetryCount - && retryCount > 0 - && !executed - && System.currentTimeMillis() < maxRetryTime); - handlePostProcessing( - executed, deleted, ignored, currentFailureCount, maxRetryCount, startTime); + } while (retryCount > 0 && status == null && System.currentTimeMillis() < maxProcessingTime); + handlePostProcessing(status == null ? TaskStatus.FAILED : status, failureCount, startTime); } finally { semaphore.release(); } diff --git a/rqueue/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessageHandler.java b/rqueue/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessageHandler.java index af2ac0e5..05a288c2 100644 --- a/rqueue/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessageHandler.java +++ b/rqueue/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessageHandler.java @@ -16,7 +16,7 @@ package com.github.sonus21.rqueue.listener; -import static org.springframework.util.Assert.notEmpty; +import static org.springframework.util.Assert.notNull; import com.github.sonus21.rqueue.annotation.RqueueListener; import com.github.sonus21.rqueue.converter.GenericMessageConverter; @@ -24,7 +24,6 @@ import com.github.sonus21.rqueue.exception.QueueDoesNotExist; import com.github.sonus21.rqueue.models.Concurrency; import com.github.sonus21.rqueue.utils.Constants; -import com.github.sonus21.rqueue.utils.MessageUtils; import com.github.sonus21.rqueue.utils.PriorityUtils; import com.github.sonus21.rqueue.utils.ValueResolver; import java.lang.reflect.Method; @@ -37,42 +36,63 @@ import java.util.List; import java.util.Map; import java.util.Set; +import org.springframework.beans.factory.config.ConfigurableBeanFactory; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ConfigurableApplicationContext; import org.springframework.core.annotation.AnnotationUtils; +import org.springframework.core.convert.ConversionService; +import org.springframework.format.support.DefaultFormattingConversionService; import org.springframework.messaging.Message; import org.springframework.messaging.MessagingException; import org.springframework.messaging.converter.CompositeMessageConverter; import org.springframework.messaging.converter.MessageConverter; import org.springframework.messaging.handler.HandlerMethod; import org.springframework.messaging.handler.annotation.support.AnnotationExceptionHandlerMethodResolver; +import org.springframework.messaging.handler.annotation.support.HeaderMethodArgumentResolver; +import org.springframework.messaging.handler.annotation.support.HeadersMethodArgumentResolver; +import org.springframework.messaging.handler.annotation.support.MessageMethodArgumentResolver; import org.springframework.messaging.handler.annotation.support.PayloadArgumentResolver; import org.springframework.messaging.handler.invocation.AbstractExceptionHandlerMethodResolver; import org.springframework.messaging.handler.invocation.AbstractMethodMessageHandler; import org.springframework.messaging.handler.invocation.HandlerMethodArgumentResolver; import org.springframework.messaging.handler.invocation.HandlerMethodReturnValueHandler; +import org.springframework.messaging.simp.annotation.support.PrincipalMethodArgumentResolver; import org.springframework.util.comparator.ComparableComparator; public class RqueueMessageHandler extends AbstractMethodMessageHandler { - private List messageConverters; + private ConversionService conversionService = new DefaultFormattingConversionService(); + private final MessageConverter messageConverter; public RqueueMessageHandler() { - messageConverters = new ArrayList<>(); - addDefaultMessageConverter(); + this(Collections.emptyList()); } public RqueueMessageHandler(List messageConverters) { - setMessageConverters(messageConverters); + notNull(messageConverters, "messageConverters cannot be null"); + List messageConverterList = new ArrayList<>(messageConverters); + messageConverterList.add(new GenericMessageConverter()); + this.messageConverter = new CompositeMessageConverter(messageConverterList); } - private void addDefaultMessageConverter() { - messageConverters.add(new GenericMessageConverter()); + private ConfigurableBeanFactory getBeanFactory() { + ApplicationContext context = getApplicationContext(); + return (context instanceof ConfigurableApplicationContext + ? ((ConfigurableApplicationContext) context).getBeanFactory() + : null); } @Override protected List initArgumentResolvers() { List resolvers = new ArrayList<>(getCustomArgumentResolvers()); - CompositeMessageConverter compositeMessageConverter = - new CompositeMessageConverter(getMessageConverters()); - resolvers.add(new PayloadArgumentResolver(compositeMessageConverter)); + // Annotation-based argument resolution + resolvers.add(new HeaderMethodArgumentResolver(this.conversionService, getBeanFactory())); + resolvers.add(new HeadersMethodArgumentResolver()); + + // Type-based argument resolution + resolvers.add(new PrincipalMethodArgumentResolver()); + resolvers.add(new MessageMethodArgumentResolver(messageConverter)); + resolvers.add(new PayloadArgumentResolver(messageConverter, null)); + return resolvers; } @@ -249,7 +269,7 @@ protected Set getDirectLookupDestinations(MappingInformation mapping) { @Override protected String getDestination(Message message) { - return (String) message.getHeaders().get(MessageUtils.getMessageHeaderKey()); + return (String) message.getHeaders().get(RqueueMessageHeaders.DESTINATION); } @Override @@ -290,13 +310,7 @@ protected void processHandlerMethodException( throw new MessagingException("An exception occurred while invoking the handler method", ex); } - public List getMessageConverters() { - return messageConverters; - } - - public void setMessageConverters(List messageConverters) { - notEmpty(messageConverters, "messageConverters list cannot be empty or null"); - this.messageConverters = messageConverters; - addDefaultMessageConverter(); + public MessageConverter getMessageConverter() { + return this.messageConverter; } } diff --git a/rqueue/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessageHeaders.java b/rqueue/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessageHeaders.java new file mode 100644 index 00000000..7689b772 --- /dev/null +++ b/rqueue/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessageHeaders.java @@ -0,0 +1,38 @@ +/* + * Copyright 2020 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.sonus21.rqueue.listener; + +import com.github.sonus21.rqueue.core.RqueueMessage; +import java.util.HashMap; +import java.util.Map; +import org.springframework.messaging.MessageHeaders; + +public class RqueueMessageHeaders { + private RqueueMessageHeaders() {} + + public static final String DESTINATION = "destination"; + public static final String ID = "messageId"; + public static final String MESSAGE = "message"; + + static MessageHeaders buildMessageHeaders(String destination, RqueueMessage rqueueMessage) { + Map headers = new HashMap<>(3); + headers.put(DESTINATION, destination); + headers.put(ID, rqueueMessage.getId()); + headers.put(MESSAGE, rqueueMessage); + return new MessageHeaders(headers); + } +} diff --git a/rqueue/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessageListenerContainer.java b/rqueue/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessageListenerContainer.java index c0a12997..7d657510 100644 --- a/rqueue/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessageListenerContainer.java +++ b/rqueue/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessageListenerContainer.java @@ -168,9 +168,6 @@ public void destroy() throws Exception { } protected void doDestroy() { - if (defaultTaskExecutor && taskExecutor != null) { - ((ThreadPoolTaskExecutor) taskExecutor).destroy(); - } Set destroyedExecutors = new HashSet<>(); for (Entry entry : queueThreadMap.entrySet()) { QueueThread queueThread = entry.getValue(); @@ -183,6 +180,12 @@ protected void doDestroy() { } } } + if (defaultTaskExecutor && taskExecutor != null) { + ThreadPoolTaskExecutor executor = (ThreadPoolTaskExecutor) taskExecutor; + if (!destroyedExecutors.contains(executor.getThreadNamePrefix())) { + executor.destroy(); + } + } } @Override @@ -243,24 +246,28 @@ public void afterPropertiesSet() throws Exception { } } } - if (QueueRegistry.getActiveQueueCount() == 0) { + List queueDetails = QueueRegistry.getActiveQueueDetails(); + if (queueDetails.isEmpty()) { return; } if (taskExecutor == null) { defaultTaskExecutor = true; - taskExecutor = createDefaultTaskExecutor(); + taskExecutor = createDefaultTaskExecutor(queueDetails); } else { - initializeDefaultExecutor(false, taskExecutor, getWorkersCount()); + initializeThreadMap(queueDetails, taskExecutor, false, queueDetails.size()); } initializeRunningQueueState(); lifecycleMgr.notifyAll(); } } - private void initializeDefaultExecutor( - boolean defaultExecutor, AsyncTaskExecutor taskExecutor, int workersCount) { + private void initializeThreadMap( + List queueDetails, + AsyncTaskExecutor taskExecutor, + boolean defaultExecutor, + int workersCount) { Semaphore semaphore = new Semaphore(workersCount); - for (QueueDetail queueDetail : QueueRegistry.getActiveQueueDetails()) { + for (QueueDetail queueDetail : queueDetails) { queueThreadMap.put( queueDetail.getName(), new QueueThread(defaultExecutor, taskExecutor, semaphore, workersCount)); @@ -273,10 +280,8 @@ private void initializeRunningQueueState() { } } - private int getWorkersCount() { - return (maxNumWorkers == null - ? QueueRegistry.getActiveQueueCount() * DEFAULT_WORKER_COUNT_PER_QUEUE - : maxNumWorkers); + private int getWorkersCount(int queueCount) { + return (maxNumWorkers == null ? queueCount * DEFAULT_WORKER_COUNT_PER_QUEUE : maxNumWorkers); } private AsyncTaskExecutor createTaskExecutor(int corePoolSize, int maxPoolSize) { @@ -286,13 +291,14 @@ private AsyncTaskExecutor createTaskExecutor(int corePoolSize, int maxPoolSize) DEFAULT_THREAD_NAME_PREFIX, prefix, corePoolSize, maxPoolSize); } - private AsyncTaskExecutor createNonConcurrencyBasedExecutor(List queueDetails) { - int workersCount = getWorkersCount(); - int maxPoolSize = workersCount + queueDetails.size(); + private AsyncTaskExecutor createNonConcurrencyBasedExecutor( + List queueDetails, int pollerCount) { + int workersCount = getWorkersCount(queueDetails.size()); + int maxPoolSize = workersCount + pollerCount; // one thread for message poller and one for executor - int corePoolSize = 2 * queueDetails.size(); + int corePoolSize = queueDetails.size() + pollerCount; AsyncTaskExecutor executor = createTaskExecutor(corePoolSize, maxPoolSize); - initializeDefaultExecutor(true, executor, workersCount); + initializeThreadMap(queueDetails, executor, true, workersCount); return executor; } @@ -305,40 +311,21 @@ private void createExecutor(QueueDetail queueDetail) { queueDetail.getName(), new QueueThread(true, executor, semaphore, concurrency.getMax())); } - public AsyncTaskExecutor createDefaultTaskExecutor() { - int withConcurrency = 0; + public AsyncTaskExecutor createDefaultTaskExecutor( + List registeredActiveQueueDetail) { List queueDetails = - QueueRegistry.getActiveQueueDetails().stream() + registeredActiveQueueDetail.stream() .filter(e -> !e.isSystemGenerated()) .collect(Collectors.toList()); - for (QueueDetail queueDetail : QueueRegistry.getActiveQueueDetails()) { - if (queueDetail.getConcurrency().getMin() > 0) { - withConcurrency += 1; - } - } - if (withConcurrency == 0) { - return createNonConcurrencyBasedExecutor(queueDetails); - } - int remainingWorkers = 0; - for (QueueDetail queueDetail : QueueRegistry.getActiveQueueDetails()) { + List withoutConcurrency = new ArrayList<>(); + for (QueueDetail queueDetail : queueDetails) { if (queueDetail.getConcurrency().getMin() > 0) { createExecutor(queueDetail); } else { - remainingWorkers += 1; - } - } - int workersCount = remainingWorkers * DEFAULT_WORKER_COUNT_PER_QUEUE; - int corePoolSize = queueDetails.size() + remainingWorkers; - int maxPoolSize = queueDetails.size() + workersCount; - AsyncTaskExecutor executor = createTaskExecutor(corePoolSize, maxPoolSize); - Semaphore semaphore = new Semaphore(workersCount); - for (QueueDetail queueDetail : queueDetails) { - if (queueDetail.getConcurrency().getMin() < 0) { - queueThreadMap.put( - queueDetail.getName(), new QueueThread(true, executor, semaphore, workersCount)); + withoutConcurrency.add(queueDetail); } } - return executor; + return createNonConcurrencyBasedExecutor(withoutConcurrency, queueDetails.size()); } private AsyncTaskExecutor createTaskExecutor( @@ -348,12 +335,18 @@ private AsyncTaskExecutor createTaskExecutor( } private List getQueueDetail(String queue, MappingInformation mappingInformation) { - int numRetries = mappingInformation.getNumRetry(); - if (!mappingInformation.getDeadLetterQueueName().isEmpty() && numRetries == -1) { - numRetries = Constants.DEFAULT_RETRY_DEAD_LETTER_QUEUE; - } else if (numRetries == -1) { - numRetries = Integer.MAX_VALUE; + int numRetry = mappingInformation.getNumRetry(); + if (!mappingInformation.getDeadLetterQueueName().isEmpty() && numRetry == -1) { + numRetry = Constants.DEFAULT_RETRY_DEAD_LETTER_QUEUE; + } else if (numRetry == -1) { + numRetry = Integer.MAX_VALUE; } + String priorityGroup = mappingInformation.getPriorityGroup(); + Map priority = mappingInformation.getPriority(); + if (StringUtils.isEmpty(priorityGroup) && priority.size() == 1) { + priorityGroup = Constants.DEFAULT_PRIORITY_GROUP; + } + QueueDetail queueDetail = QueueDetail.builder() .name(queue) @@ -366,9 +359,9 @@ private List getQueueDetail(String queue, MappingInformation mappin .visibilityTimeout(mappingInformation.getVisibilityTimeout()) .concurrency(mappingInformation.getConcurrency()) .active(mappingInformation.isActive()) - .numRetry(numRetries) - .priority(mappingInformation.getPriority()) - .priorityGroup(mappingInformation.getPriorityGroup()) + .numRetry(numRetry) + .priority(priority) + .priorityGroup(priorityGroup) .build(); if (queueDetail.getPriority().size() <= 1) { return Collections.singletonList(queueDetail); @@ -584,23 +577,23 @@ public void setPreExecutionMessageProcessor(MessageProcessor preExecutionMessage this.preExecutionMessageProcessor = preExecutionMessageProcessor; } + public TaskExecutionBackOff getTaskExecutionBackOff() { + return taskExecutionBackOff; + } + public void setTaskExecutionBackOff(TaskExecutionBackOff taskExecutionBackOff) { notNull(taskExecutionBackOff, "taskExecutionBackOff cannot be null"); this.taskExecutionBackOff = taskExecutionBackOff; } - public TaskExecutionBackOff getTaskExecutionBackOff() { - return taskExecutionBackOff; + public PriorityMode getPriorityMode() { + return priorityMode; } public void setPriorityMode(PriorityMode priorityMode) { this.priorityMode = priorityMode; } - public PriorityMode getPriorityMode() { - return priorityMode; - } - RqueueCounter getRqueueCounter() { return rqueueCounter; } diff --git a/rqueue/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessagePoller.java b/rqueue/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessagePoller.java index b7ef1833..11cf899b 100644 --- a/rqueue/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessagePoller.java +++ b/rqueue/src/main/java/com/github/sonus21/rqueue/listener/RqueueMessagePoller.java @@ -28,17 +28,9 @@ @Slf4j abstract class RqueueMessagePoller extends MessageContainerBase { - List queues; - - enum DeactivateType { - POLL_FAILED, - NO_MESSAGE, - SEMAPHORE_EXCEPTION, - SEMAPHORE_UNAVAILABLE, - } - private final TaskExecutionBackOff taskBackOff; private final int retryPerPoll; + List queues; RqueueMessagePoller( String groupName, @@ -46,7 +38,7 @@ enum DeactivateType { TaskExecutionBackOff taskExecutionBackOff, int retryPerPoll) { super(log, groupName, container); - taskBackOff = taskExecutionBackOff; + this.taskBackOff = taskExecutionBackOff; this.retryPerPoll = retryPerPoll; } @@ -125,4 +117,11 @@ void poll(int index, String queue, QueueDetail queueDetail, QueueThread queueThr abstract long getSemaphoreWaiTime(); abstract void deactivate(int index, String queue, DeactivateType deactivateType); + + enum DeactivateType { + POLL_FAILED, + NO_MESSAGE, + SEMAPHORE_EXCEPTION, + SEMAPHORE_UNAVAILABLE, + } } diff --git a/rqueue/src/main/java/com/github/sonus21/rqueue/listener/StrictPriorityPoller.java b/rqueue/src/main/java/com/github/sonus21/rqueue/listener/StrictPriorityPoller.java index a18b390f..8994a4d7 100644 --- a/rqueue/src/main/java/com/github/sonus21/rqueue/listener/StrictPriorityPoller.java +++ b/rqueue/src/main/java/com/github/sonus21/rqueue/listener/StrictPriorityPoller.java @@ -32,9 +32,9 @@ class StrictPriorityPoller extends RqueueMessagePoller { private final Map queueNameToDetail; - private Map lastFetchedTime = new HashMap<>(); private final Map queueNameToThread; private final Map queueDeactivationTime = new HashMap<>(); + private Map lastFetchedTime = new HashMap<>(); StrictPriorityPoller( String groupName, @@ -68,11 +68,11 @@ private String getQueueToPoll() { } for (String queue : queues) { if (isQueueActive(queue)) { - Long lastDeactivationTime = queueDeactivationTime.get(queue); - if (lastDeactivationTime == null) { + Long deactivationTime = queueDeactivationTime.get(queue); + if (deactivationTime == null) { return queue; } - if (now - lastDeactivationTime > getPollingInterval()) { + if (now - deactivationTime > getPollingInterval()) { return queue; } } diff --git a/rqueue/src/main/java/com/github/sonus21/rqueue/listener/WeightedPriorityPoller.java b/rqueue/src/main/java/com/github/sonus21/rqueue/listener/WeightedPriorityPoller.java index 4671abe3..25a1ba13 100644 --- a/rqueue/src/main/java/com/github/sonus21/rqueue/listener/WeightedPriorityPoller.java +++ b/rqueue/src/main/java/com/github/sonus21/rqueue/listener/WeightedPriorityPoller.java @@ -28,9 +28,9 @@ import org.slf4j.event.Level; class WeightedPriorityPoller extends RqueueMessagePoller { - private List queueDetailList; private final Map queueNameToThread; private final Map queueNameToDetail; + private List queueDetailList; private int[] currentWeight; private int[] weight; private float[] probability; diff --git a/rqueue/src/main/java/com/github/sonus21/rqueue/models/Concurrency.java b/rqueue/src/main/java/com/github/sonus21/rqueue/models/Concurrency.java index 008956d5..0df0b970 100644 --- a/rqueue/src/main/java/com/github/sonus21/rqueue/models/Concurrency.java +++ b/rqueue/src/main/java/com/github/sonus21/rqueue/models/Concurrency.java @@ -17,10 +17,12 @@ package com.github.sonus21.rqueue.models; import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; import lombok.Getter; @Getter @AllArgsConstructor +@EqualsAndHashCode public class Concurrency { private int min; private int max; @@ -28,12 +30,4 @@ public class Concurrency { public MinMax toMinMax() { return new MinMax<>(min, max); } - - @Override - public boolean equals(Object other) { - if (other instanceof Concurrency) { - return ((Concurrency) other).max == this.max && ((Concurrency) other).min == this.min; - } - return false; - } } diff --git a/rqueue/src/main/java/com/github/sonus21/rqueue/models/aggregator/TasksStat.java b/rqueue/src/main/java/com/github/sonus21/rqueue/models/aggregator/TasksStat.java index b4d8815e..9d6a901b 100644 --- a/rqueue/src/main/java/com/github/sonus21/rqueue/models/aggregator/TasksStat.java +++ b/rqueue/src/main/java/com/github/sonus21/rqueue/models/aggregator/TasksStat.java @@ -17,7 +17,9 @@ package com.github.sonus21.rqueue.models.aggregator; import com.github.sonus21.rqueue.models.db.JobRunTime; +import lombok.ToString; +@ToString public class TasksStat { public long discarded = 0; public long success = 0; diff --git a/rqueue/src/main/java/com/github/sonus21/rqueue/models/db/TaskStatus.java b/rqueue/src/main/java/com/github/sonus21/rqueue/models/db/TaskStatus.java index cda5f941..486174b9 100644 --- a/rqueue/src/main/java/com/github/sonus21/rqueue/models/db/TaskStatus.java +++ b/rqueue/src/main/java/com/github/sonus21/rqueue/models/db/TaskStatus.java @@ -31,7 +31,9 @@ public enum TaskStatus { DELETED("Message deleted", false), DISCARDED("Message discarded", true), RETRIED("Retired at least once", true), - IGNORED("Ignored task", false); + FAILED("failed", false), + IGNORED("Ignored task", false), + QUEUE_INACTIVE("Queue inactive", false); private String description; private boolean chartEnabled; diff --git a/rqueue/src/main/java/com/github/sonus21/rqueue/utils/Constants.java b/rqueue/src/main/java/com/github/sonus21/rqueue/utils/Constants.java index 86eeb6d4..97630193 100644 --- a/rqueue/src/main/java/com/github/sonus21/rqueue/utils/Constants.java +++ b/rqueue/src/main/java/com/github/sonus21/rqueue/utils/Constants.java @@ -18,8 +18,6 @@ public class Constants { - private Constants() {} - public static final String BLANK = ""; public static final long ONE_MILLI = 1000; public static final int ONE_MILLI_INT = 1000; @@ -42,7 +40,10 @@ private Constants() {} public static final int MAX_MESSAGES = 100; public static final int DEFAULT_WORKER_COUNT_PER_QUEUE = 2; public static final int AGGREGATION_LOCK_DURATION_IN_SECONDS = 5; - public static final String MAVEN_REPO_LINK = - "https://repo1.maven.org/maven2/com/github/sonus21/rqueue"; + public static final String GITHUB_API_FOR_LATEST_RELEASE = + "https://api.github.com/repos/sonus21/rqueue/releases/latest"; public static final String DEFAULT_PRIORITY_KEY = "DEFAULT_PRIORITY"; + public static final String DEFAULT_PRIORITY_GROUP = "Default"; + + private Constants() {} } diff --git a/rqueue/src/main/java/com/github/sonus21/rqueue/utils/HttpUtils.java b/rqueue/src/main/java/com/github/sonus21/rqueue/utils/HttpUtils.java index 0687f5a7..1f9fbc12 100644 --- a/rqueue/src/main/java/com/github/sonus21/rqueue/utils/HttpUtils.java +++ b/rqueue/src/main/java/com/github/sonus21/rqueue/utils/HttpUtils.java @@ -25,14 +25,14 @@ public class HttpUtils { private HttpUtils() {} - public static String readUrl(String url) { + public static T readUrl(String url, Class clazz) { try { RestTemplate restTemplate = new RestTemplate(); SimpleClientHttpRequestFactory rf = (SimpleClientHttpRequestFactory) restTemplate.getRequestFactory(); rf.setReadTimeout(2 * Constants.ONE_MILLI_INT); rf.setConnectTimeout(2 * Constants.ONE_MILLI_INT); - return restTemplate.getForObject(url, String.class); + return restTemplate.getForObject(url, clazz); } catch (Exception e) { log.error("GET call failed for {}", url, e); return null; diff --git a/rqueue/src/main/java/com/github/sonus21/rqueue/utils/MessageUtils.java b/rqueue/src/main/java/com/github/sonus21/rqueue/utils/MessageUtils.java index d71e00ff..7c3ce260 100644 --- a/rqueue/src/main/java/com/github/sonus21/rqueue/utils/MessageUtils.java +++ b/rqueue/src/main/java/com/github/sonus21/rqueue/utils/MessageUtils.java @@ -19,17 +19,14 @@ import static org.springframework.util.Assert.notEmpty; import com.github.sonus21.rqueue.core.RqueueMessage; -import java.util.Collections; import java.util.List; -import java.util.Map; import org.springframework.messaging.Message; import org.springframework.messaging.converter.MessageConversionException; import org.springframework.messaging.converter.MessageConverter; import org.springframework.messaging.support.GenericMessage; public class MessageUtils { - private static final String MESSAGE_HEADER_KEY = "Rqueue"; - private static final String MESSAGE_META_DATA_KEY_PREFIX = "__rq::m-mdata::"; + private static final String META_DATA_KEY_PREFIX = "__rq::m-mdata::"; public MessageUtils() {} @@ -68,15 +65,7 @@ public static Object convertMessageToObject( return null; } - public static String getMessageHeaderKey() { - return MESSAGE_HEADER_KEY; - } - - public static Map getMessageHeader(String queueName) { - return Collections.singletonMap(getMessageHeaderKey(), queueName); - } - public static String getMessageMetaId(String messageId) { - return MESSAGE_META_DATA_KEY_PREFIX + messageId; + return META_DATA_KEY_PREFIX + messageId; } } diff --git a/rqueue/src/main/java/com/github/sonus21/rqueue/utils/RedisUtils.java b/rqueue/src/main/java/com/github/sonus21/rqueue/utils/RedisUtils.java index 4d81c61f..37b98082 100644 --- a/rqueue/src/main/java/com/github/sonus21/rqueue/utils/RedisUtils.java +++ b/rqueue/src/main/java/com/github/sonus21/rqueue/utils/RedisUtils.java @@ -56,11 +56,11 @@ public static List executePipeLine( }); } - public interface RedisPipelineCallback { - void doInRedis( - RedisConnection connection, - StringRedisSerializer keySerializer, - RqueueRedisSerializer valueSerializer); + public static void setVersion( + RedisConnectionFactory connectionFactory, String versionKey, int version) { + RedisConnection connection = RedisConnectionUtils.getConnection(connectionFactory); + byte[] versionKeyBytes = versionKey.getBytes(); + connection.set(versionKeyBytes, String.valueOf(version).getBytes()); } public static int updateAndGetVersion( @@ -82,4 +82,11 @@ public static int updateAndGetVersion( } return Integer.parseInt(new String(versionFromDb)); } + + public interface RedisPipelineCallback { + void doInRedis( + RedisConnection connection, + StringRedisSerializer keySerializer, + RqueueRedisSerializer valueSerializer); + } } diff --git a/rqueue/src/main/java/com/github/sonus21/rqueue/utils/StringUtils.java b/rqueue/src/main/java/com/github/sonus21/rqueue/utils/StringUtils.java index dcec34e3..a4dc26a3 100644 --- a/rqueue/src/main/java/com/github/sonus21/rqueue/utils/StringUtils.java +++ b/rqueue/src/main/java/com/github/sonus21/rqueue/utils/StringUtils.java @@ -39,23 +39,36 @@ public static boolean isAlpha(Character c) { return Character.isUpperCase(c) || Character.isLowerCase(c); } - public static String convertToCamelCase(String queueName) { + public static String getBeanName(String queueName) { + return convertToCamelCase(queueName); + } + + public static String convertToCamelCase(String string) { + String txt = clean(string); + if (isEmpty(txt)) { + throw new IllegalArgumentException("string is empty"); + } StringBuilder sb = new StringBuilder(); - for (int i = 0; i < queueName.length(); i++) { - Character c = queueName.charAt(i); + for (int i = 0; i < txt.length(); i++) { + char c = txt.charAt(i); if (isAlpha(c)) { - if (i > 0 && !isAlpha(queueName.charAt(i - 1))) { + if (i == 0) { + sb.append(c); + } else if (!isAlpha(txt.charAt(i - 1))) { sb.append(Character.toUpperCase(c)); - } else { + } else if (Character.isLowerCase(c) + || (Character.isUpperCase(c) && Character.isLowerCase(txt.charAt(i - 1)))) { sb.append(c); + } else { + sb.append(Character.toLowerCase(c)); } } } - String txt = sb.toString(); - if (txt.isEmpty()) { + String convertedTxt = sb.toString(); + if (convertedTxt.isEmpty()) { return txt; } - return Introspector.decapitalize(txt); + return Introspector.decapitalize(convertedTxt); } public static String groupName(String name) { diff --git a/rqueue/src/main/java/com/github/sonus21/rqueue/utils/ThreadUtils.java b/rqueue/src/main/java/com/github/sonus21/rqueue/utils/ThreadUtils.java index be9f699b..d4cb8261 100644 --- a/rqueue/src/main/java/com/github/sonus21/rqueue/utils/ThreadUtils.java +++ b/rqueue/src/main/java/com/github/sonus21/rqueue/utils/ThreadUtils.java @@ -114,7 +114,7 @@ public static boolean waitForWorkerTermination( } public static String getWorkerName(String name) { - String camelCase = StringUtils.convertToCamelCase(name); + String camelCase = StringUtils.getBeanName(name); return camelCase + "Consumer"; } diff --git a/rqueue/src/main/java/com/github/sonus21/rqueue/utils/backoff/ExponentialTaskExecutionBackOff.java b/rqueue/src/main/java/com/github/sonus21/rqueue/utils/backoff/ExponentialTaskExecutionBackOff.java index 46d2f859..4c391761 100644 --- a/rqueue/src/main/java/com/github/sonus21/rqueue/utils/backoff/ExponentialTaskExecutionBackOff.java +++ b/rqueue/src/main/java/com/github/sonus21/rqueue/utils/backoff/ExponentialTaskExecutionBackOff.java @@ -104,29 +104,49 @@ private void checkMultiplier(double multiplier) { } } - /** Set the initial interval in milliseconds. */ + /** + * Return the initial interval in milliseconds. + * + * @return configured initial interval. + */ + public long getInitialInterval() { + return this.initialInterval; + } + + /** + * Set the initial interval in milliseconds. + * + * @param initialInterval initial interval in milli seconds. + */ public void setInitialInterval(long initialInterval) { checkInitialInterval(initialInterval); this.initialInterval = initialInterval; } - /** Return the initial interval in milliseconds. */ - public long getInitialInterval() { - return this.initialInterval; + /** + * Return the maximum number of retries. + * + * @return maximum retries that will be performed. + */ + public int getMaxRetries() { + return this.maxRetries; } - /** Set the maximum number of retries */ + /** + * Set the maximum number of retries + * + * @param maxRetries maximum retries + */ public void setMaxRetries(int maxRetries) { checkMaxRetries(maxRetries); this.maxRetries = maxRetries; } - /** Return the maximum number of retries. */ - public int getMaxRetries() { - return this.maxRetries; - } - - /** get the multiplier */ + /** + * get the multiplier + * + * @return multiplier for this back off. + */ public double getMultiplier() { return multiplier; } @@ -160,6 +180,7 @@ public void setMaxInterval(long maxInterval) { this.maxInterval = maxInterval; } + /** {@inheritDoc} */ @Override public long nextBackOff(Object object, RqueueMessage rqueueMessage, int failureCount) { if (failureCount >= getMaxRetries()) { diff --git a/rqueue/src/main/java/com/github/sonus21/rqueue/utils/backoff/FixedTaskExecutionBackOff.java b/rqueue/src/main/java/com/github/sonus21/rqueue/utils/backoff/FixedTaskExecutionBackOff.java index 5412be5b..9ac9e870 100644 --- a/rqueue/src/main/java/com/github/sonus21/rqueue/utils/backoff/FixedTaskExecutionBackOff.java +++ b/rqueue/src/main/java/com/github/sonus21/rqueue/utils/backoff/FixedTaskExecutionBackOff.java @@ -55,15 +55,24 @@ public FixedTaskExecutionBackOff(long interval, int maxRetries) { this.maxRetries = maxRetries; } + /** + * Return the delay between two attempts in milliseconds. + * + * @return the configured interval in milli seconds. + */ + public long getInterval() { + return this.interval; + } + /** Set the delay between two attempts in milliseconds. */ public void setInterval(long interval) { checkInterval(interval); this.interval = interval; } - /** Return the delay between two attempts in milliseconds. */ - public long getInterval() { - return this.interval; + /** Return the maximum number of retries. */ + public int getMaxRetries() { + return this.maxRetries; } /** Set the maximum number of retries */ @@ -72,11 +81,6 @@ public void setMaxRetries(int maxRetries) { this.maxRetries = maxRetries; } - /** Return the maximum number of retries. */ - public int getMaxRetries() { - return this.maxRetries; - } - @Override public long nextBackOff(Object object, RqueueMessage rqueueMessage, int failureCount) { if (failureCount >= getMaxRetries()) { diff --git a/rqueue/src/main/java/com/github/sonus21/rqueue/web/controller/RqueueViewController.java b/rqueue/src/main/java/com/github/sonus21/rqueue/web/controller/RqueueViewController.java index e58ea506..ebd54606 100644 --- a/rqueue/src/main/java/com/github/sonus21/rqueue/web/controller/RqueueViewController.java +++ b/rqueue/src/main/java/com/github/sonus21/rqueue/web/controller/RqueueViewController.java @@ -24,7 +24,6 @@ import com.github.sonus21.rqueue.models.enums.DataType; import com.github.sonus21.rqueue.models.enums.NavTab; import com.github.sonus21.rqueue.models.response.RedisDataDetail; -import com.github.sonus21.rqueue.utils.Constants; import com.github.sonus21.rqueue.web.service.RqueueQDetailService; import com.github.sonus21.rqueue.web.service.RqueueSystemManagerService; import com.github.sonus21.rqueue.web.service.RqueueUtilityService; @@ -37,11 +36,11 @@ import java.util.List; import java.util.Locale; import java.util.Map.Entry; -import java.util.stream.Collectors; import javax.servlet.http.HttpServletResponse; import org.jtwig.spring.JtwigViewResolver; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.data.util.Pair; import org.springframework.stereotype.Controller; import org.springframework.ui.Model; import org.springframework.web.bind.annotation.GetMapping; @@ -83,8 +82,9 @@ private void addNavData(Model model, NavTab tab) { } private void addBasicDetails(Model model) { - model.addAttribute("latestVersion", rqueueUtilityService.getLatestVersion()); - model.addAttribute("mavenRepoLink", Constants.MAVEN_REPO_LINK); + Pair releaseAndVersion = rqueueUtilityService.getLatestVersion(); + model.addAttribute("releaseLink", releaseAndVersion.getFirst()); + model.addAttribute("latestVersion", releaseAndVersion.getSecond()); model.addAttribute( "time", OffsetDateTime.now(ZoneOffset.UTC).format(DateTimeFormatter.ISO_LOCAL_DATE_TIME)); model.addAttribute("timeInMilli", System.currentTimeMillis()); @@ -114,10 +114,7 @@ public View queues(Model model, HttpServletResponse response) throws Exception { addBasicDetails(model); addNavData(model, NavTab.QUEUES); model.addAttribute("title", "Queues"); - List queueConfigs = - rqueueSystemManagerService.getQueueConfigs().stream() - .sorted(Comparator.comparing(QueueConfig::getName)) - .collect(Collectors.toList()); + List queueConfigs = rqueueSystemManagerService.getSortedQueueConfigs(); List>>> queueNameConfigs = new ArrayList<>(rqueueQDetailService.getQueueDataStructureDetails(queueConfigs).entrySet()); queueNameConfigs.sort(Comparator.comparing(Entry::getKey)); diff --git a/rqueue/src/main/java/com/github/sonus21/rqueue/web/service/RqueueSystemManagerService.java b/rqueue/src/main/java/com/github/sonus21/rqueue/web/service/RqueueSystemManagerService.java index 9bd8e3e7..189b90db 100644 --- a/rqueue/src/main/java/com/github/sonus21/rqueue/web/service/RqueueSystemManagerService.java +++ b/rqueue/src/main/java/com/github/sonus21/rqueue/web/service/RqueueSystemManagerService.java @@ -30,5 +30,7 @@ public interface RqueueSystemManagerService { List getQueueConfigs(); + List getSortedQueueConfigs(); + QueueConfig getQueueConfig(String queueName); } diff --git a/rqueue/src/main/java/com/github/sonus21/rqueue/web/service/RqueueTaskAggregatorService.java b/rqueue/src/main/java/com/github/sonus21/rqueue/web/service/RqueueTaskAggregatorService.java index 5a155e3b..58de0e3a 100644 --- a/rqueue/src/main/java/com/github/sonus21/rqueue/web/service/RqueueTaskAggregatorService.java +++ b/rqueue/src/main/java/com/github/sonus21/rqueue/web/service/RqueueTaskAggregatorService.java @@ -43,7 +43,7 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Future; -import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.DisposableBean; @@ -101,7 +101,7 @@ public void start() { } this.eventAggregatorTasks = new ArrayList<>(); this.queueNameToEvents = new ConcurrentHashMap<>(); - this.queue = new LinkedBlockingDeque<>(); + this.queue = new LinkedBlockingQueue<>(); int threadCount = rqueueWebConfig.getStatsAggregatorThreadCount(); this.taskExecutor = ThreadUtils.createTaskScheduler(threadCount, "RqueueTaskAggregator-", 30); for (int i = 0; i < threadCount; i++) { diff --git a/rqueue/src/main/java/com/github/sonus21/rqueue/web/service/RqueueUtilityService.java b/rqueue/src/main/java/com/github/sonus21/rqueue/web/service/RqueueUtilityService.java index a5f1f1d2..86cfbbe8 100644 --- a/rqueue/src/main/java/com/github/sonus21/rqueue/web/service/RqueueUtilityService.java +++ b/rqueue/src/main/java/com/github/sonus21/rqueue/web/service/RqueueUtilityService.java @@ -20,6 +20,7 @@ import com.github.sonus21.rqueue.models.response.BooleanResponse; import com.github.sonus21.rqueue.models.response.MessageMoveResponse; import com.github.sonus21.rqueue.models.response.StringResponse; +import org.springframework.data.util.Pair; public interface RqueueUtilityService { @@ -29,7 +30,7 @@ public interface RqueueUtilityService { BooleanResponse deleteQueueMessages(String queueName, int remainingMessage); - String getLatestVersion(); + Pair getLatestVersion(); StringResponse getDataType(String name); } diff --git a/rqueue/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueDashboardChartServiceImpl.java b/rqueue/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueDashboardChartServiceImpl.java index a11ff724..745c6d05 100644 --- a/rqueue/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueDashboardChartServiceImpl.java +++ b/rqueue/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueDashboardChartServiceImpl.java @@ -59,7 +59,8 @@ public class RqueueDashboardChartServiceImpl implements RqueueDashboardChartServ @Autowired public RqueueDashboardChartServiceImpl( RqueueQStatsDao rqueueQStatsDao, - RqueueConfig rqueueConfig, RqueueWebConfig rqueueWebConfig, + RqueueConfig rqueueConfig, + RqueueWebConfig rqueueWebConfig, RqueueSystemManagerService rqueueSystemManagerService) { this.rqueueQStatsDao = rqueueQStatsDao; this.rqueueConfig = rqueueConfig; diff --git a/rqueue/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueQDetailServiceImpl.java b/rqueue/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueQDetailServiceImpl.java index 15da0540..98db60a1 100644 --- a/rqueue/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueQDetailServiceImpl.java +++ b/rqueue/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueQDetailServiceImpl.java @@ -42,7 +42,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -316,7 +315,7 @@ private void setHeadersIfRequired( @Override public List> getRunningTasks() { - List queueConfigs = rqueueSystemManagerService.getQueueConfigs(); + List queueConfigs = rqueueSystemManagerService.getSortedQueueConfigs(); List> rows = new ArrayList<>(); List result = new ArrayList<>(); if (!CollectionUtils.isEmpty(queueConfigs)) { @@ -341,7 +340,7 @@ public List> getRunningTasks() { @Override public List> getWaitingTasks() { - List queueConfigs = rqueueSystemManagerService.getQueueConfigs(); + List queueConfigs = rqueueSystemManagerService.getSortedQueueConfigs(); List> rows = new ArrayList<>(); List result = new ArrayList<>(); if (!CollectionUtils.isEmpty(queueConfigs)) { @@ -364,7 +363,7 @@ public List> getWaitingTasks() { @Override public List> getScheduledTasks() { - List queueConfigs = rqueueSystemManagerService.getQueueConfigs(); + List queueConfigs = rqueueSystemManagerService.getSortedQueueConfigs(); List> rows = new ArrayList<>(); List result = new ArrayList<>(); if (!CollectionUtils.isEmpty(queueConfigs)) { @@ -410,7 +409,7 @@ private void addRows( @Override public List> getDeadLetterTasks() { - List queueConfigs = rqueueSystemManagerService.getQueueConfigs(); + List queueConfigs = rqueueSystemManagerService.getSortedQueueConfigs(); List> queueConfigAndDlq = new ArrayList<>(); for (QueueConfig queueConfig : queueConfigs) { if (queueConfig.hasDeadLetterQueue()) { diff --git a/rqueue/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueSystemManagerServiceImpl.java b/rqueue/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueSystemManagerServiceImpl.java index aa775cc0..ca27a7c7 100644 --- a/rqueue/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueSystemManagerServiceImpl.java +++ b/rqueue/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueSystemManagerServiceImpl.java @@ -32,6 +32,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -193,6 +194,14 @@ public List getQueueConfigs() { return getQueueConfigs(queues); } + @Override + public List getSortedQueueConfigs() { + List queues = getQueues(); + return getQueueConfigs(queues).stream() + .sorted(Comparator.comparing(QueueConfig::getName)) + .collect(Collectors.toList()); + } + @Override public QueueConfig getQueueConfig(String queueName) { List queueConfigs = getQueueConfigs(Collections.singletonList(queueName)); diff --git a/rqueue/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueUtilityServiceImpl.java b/rqueue/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueUtilityServiceImpl.java index e83d8baf..d7ed5ca9 100644 --- a/rqueue/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueUtilityServiceImpl.java +++ b/rqueue/src/main/java/com/github/sonus21/rqueue/web/service/impl/RqueueUtilityServiceImpl.java @@ -36,12 +36,12 @@ import com.github.sonus21.rqueue.web.service.RqueueMessageMetadataService; import com.github.sonus21.rqueue.web.service.RqueueUtilityService; import java.time.Duration; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; +import java.util.LinkedHashMap; +import java.util.Map; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.data.util.Pair; import org.springframework.stereotype.Service; @Service @@ -54,6 +54,7 @@ public class RqueueUtilityServiceImpl implements RqueueUtilityService { private final RqueueMessageMetadataService messageMetadataService; private final RqueueConfig rqueueConfig; private String latestVersion = "NA"; + private String releaseLink = "#"; private long versionFetchTime = 0; @Autowired @@ -167,22 +168,23 @@ public BooleanResponse deleteQueueMessages(String queueName, int remainingMessag } @Override - public String getLatestVersion() { + @SuppressWarnings("unchecked") + public Pair getLatestVersion() { if (System.currentTimeMillis() - versionFetchTime > Constants.MILLIS_IN_A_DAY) { - String response = readUrl(Constants.MAVEN_REPO_LINK + "/maven-metadata.xml"); + Map response = + readUrl(Constants.GITHUB_API_FOR_LATEST_RELEASE, LinkedHashMap.class); if (response != null) { - List lines = - Arrays.stream(response.split("\n")) - .map(String::trim) - .filter(e -> e.startsWith("")) - .collect(Collectors.toList()); - if (!lines.isEmpty()) { - latestVersion = lines.get(0); - versionFetchTime = System.currentTimeMillis(); + String tagName = (String) response.get("tag_name"); + if (tagName != null && !tagName.isEmpty()) { + if (Character.toLowerCase(tagName.charAt(0)) == 'v') { + releaseLink = (String) response.get("html_url"); + latestVersion = tagName.substring(1); + versionFetchTime = System.currentTimeMillis(); + } } } } - return latestVersion; + return Pair.of(releaseLink, latestVersion); } @Override diff --git a/rqueue/src/main/resources/public/rqueue/css/rqueue.css b/rqueue/src/main/resources/public/rqueue/css/rqueue.css index 8a0ffd10..1b16f764 100644 --- a/rqueue/src/main/resources/public/rqueue/css/rqueue.css +++ b/rqueue/src/main/resources/public/rqueue/css/rqueue.css @@ -507,6 +507,10 @@ section { background: #ee3535; } -.btn-delete-all{ +.btn-delete-all { margin-left: 10px; +} + +.delete-message-btn { + padding: 0 10px; } \ No newline at end of file diff --git a/rqueue/src/main/resources/public/rqueue/js/rqueue.js b/rqueue/src/main/resources/public/rqueue/js/rqueue.js index a4572320..20f428c8 100644 --- a/rqueue/src/main/resources/public/rqueue/js/rqueue.js +++ b/rqueue/src/main/resources/public/rqueue/js/rqueue.js @@ -21,17 +21,7 @@ var dataName = null; var dataType = null; var currentPage = 0; var defaultPageSize = null; -var deleteAllKey = null; - -function drawChartElement(data, options, div_id) { - google.charts.load('current', {'packages': ['corechart']}); - google.charts.setOnLoadCallback(function () { - var chart = new google.visualization.LineChart( - document.getElementById(div_id)); - var chartData = google.visualization.arrayToDataTable(data, false); - chart.draw(chartData, options); - }); -} +var deleteButtonId = null; function showError(message) { $("#global-error-container").show(); @@ -51,6 +41,27 @@ function ajaxRequest(url, type, payload, successHandler, failureHandler) { }); } +function getSelectedOption(id) { + return $('#' + id + ' :selected').val(); +} + +function json(data) { + return JSON.stringify(data); +} + +//==================================================== +// Queue Charts +//==================================================== +function drawChartElement(data, options, div_id) { + google.charts.load('current', {'packages': ['corechart']}); + google.charts.setOnLoadCallback(function () { + var chart = new google.visualization.LineChart( + document.getElementById(div_id)); + var chartData = google.visualization.arrayToDataTable(data, false); + chart.draw(chartData, options); + }); +} + function drawChart(payload, div_id) { ajaxRequest('/rqueue/api/v1/chart', 'POST', payload, function (response) { @@ -71,10 +82,6 @@ function drawChart(payload, div_id) { }); } -function getSelectedOption(id) { - return $('#' + id + ' :selected').val(); -} - function refreshStatsChart(chartParams, div_id) { var types = []; var aggregatorType = $('#stats-aggregator-type :selected').val(); @@ -92,6 +99,10 @@ function refreshLatencyChart(chartParams, div_id) { drawChart(chartParams, div_id); } +//================================================================== +// Data Exploration +//================================================================== + function exploreData(e) { var element = $(e); dataName = element.data('name'); @@ -99,10 +110,6 @@ function exploreData(e) { dataKey = element.data('key'); } -function json(data) { - return JSON.stringify(data); -} - function displayTable(nextOrPrev) { var pageSize = parseInt($('#page-size').val()); var pageNumber = currentPage; @@ -165,7 +172,7 @@ function displayTable(nextOrPrev) { tds += ("" + row[j] + ""); } if (row[row.length - 1] === 'DELETE') { - tds += ("Delete "); + tds += ("Delete "); } else { tds += ("" + row[j] + ""); } @@ -180,35 +187,6 @@ function displayTable(nextOrPrev) { }); } -function deleteAll() { - deleteAllKey = queueName; - $('#delete-modal-body').empty().append( - "Do you really want to delete Queue: " + queueName - + " ? This process cannot be undone."); - $('#delete-modal').modal('show'); -} - -function deleteMessage(e) { - var id = e.parentNode.parentNode.children[0].textContent; - var url = '/rqueue/api/v1/data-set/' + queueName + "/" + id; - $.ajax({ - url: url, - type: 'DELETE', - success: function (response) { - if (response.code === 0) { - refreshPage(); - } else { - alert("Failed:" + response.message + " Please retry!"); - console.log(response); - } - }, - fail: function (response) { - console.log('failed, ' + response); - showError("Something went wrong! Please reload!"); - } - }); -} - function refreshPage() { displayTable(null); } @@ -262,62 +240,9 @@ $('#explore-queue').on('hidden.bs.modal', function () { $('#page-size').val(defaultPageSize); }); -$('.delete-queue').on("click", function () { - queueName = $(this).data('name'); - $('#delete-modal-body').empty().append( - "Do you really want to delete Queue: ?" + queueName - + "    This process cannot be undone."); - $('#delete-modal').modal('show'); -}); - -function makeQueueEmpty() { - $.ajax({ - url: "/rqueue/api/v1/data-set/" + deleteAllKey, - type: 'DELETE', - success: function (response) { - if (response.code === 0) { - currentPage = 0; - refreshPage(); - } else { - alert("Failed:" + response.message + " Please retry!"); - console.log(response); - } - }, - fail: function (response) { - console.log('failed, ' + response); - showError("Something went wrong! Please reload!"); - } - }); -} - -function deleteQueue() { - $.ajax({ - url: "/rqueue/api/v1/queues/" + queueName, - type: 'DELETE', - success: function (response) { - if (response.code === 0) { - window.location.replace(window.location.href); - } else { - alert("Failed:" + response.message + " Please retry!"); - console.log(response); - } - }, - fail: function (response) { - console.log('failed, ' + response); - showError("Something went wrong! Please reload!"); - } - }) -} - -$('.delete-btn').on("click", function () { - if (queueName !== null) { - deleteQueue(); - } else if (deleteAllKey !== null) { - makeQueueEmpty(); - } else { - console.log("Something is not correct :("); - } -}); +//============================================================== +// Message Move form +//============================================================== function enableKeyForm() { var dataType = $('#data-name-type').text(); @@ -413,4 +338,108 @@ $('#move-button').on("click", function () { console.log('failed, ' + response); showError("Something wet wrong! Please retry!"); }); -}); \ No newline at end of file +}); + +//================================================ +// Delete Handler +//================================================= + + +function deleteModalBody() { + if (deleteButtonId === "DELETE_ALL" && dataName !== undefined) { + return "Do you really want to delete Queue: " + dataName + + " ? This process cannot be undone."; + } else if (deleteButtonId === "DELETE_QUEUE" && queueName !== undefined) { + return "Do you really want to delete Queue: " + queueName + + " ? This process cannot be undone."; + } + throw deleteButtonId; +} + +function updateDeleteModal() { + $('#delete-modal-body').empty().append(deleteModalBody()); + $('#delete-modal').modal('show'); +} + +function deleteMessage(e) { + var id = e.parentNode.parentNode.children[0].textContent; + var url = '/rqueue/api/v1/data-set/' + queueName + "/" + id; + $.ajax({ + url: url, + type: 'DELETE', + success: function (response) { + if (response.code === 0) { + refreshPage(); + } else { + alert("Failed:" + response.message + " Please retry!"); + console.log(response); + } + }, + fail: function (response) { + console.log('failed, ' + response); + showError("Something went wrong! Please reload!"); + } + }); +} + +function deleteAll() { + deleteButtonId = "DELETE_ALL"; + updateDeleteModal(); +} + +$('.delete-queue').on("click", function () { + deleteButtonId = "DELETE_QUEUE"; + queueName = $(this).data('name'); + updateDeleteModal(); +}); + +function makeQueueEmpty() { + $.ajax({ + url: "/rqueue/api/v1/data-set/" + dataName, + type: 'DELETE', + success: function (response) { + if (response.code === 0) { + currentPage = 0; + refreshPage(); + } else { + alert("Failed:" + response.message + " Please retry!"); + console.log(response); + } + }, + fail: function (response) { + console.log('failed, ' + response); + showError("Something went wrong! Please reload!"); + } + }); +} + +function deleteQueue() { + $.ajax({ + url: "/rqueue/api/v1/queues/" + queueName, + type: 'DELETE', + success: function (response) { + if (response.code === 0) { + window.location.replace(window.location.href); + } else { + alert("Failed:" + response.message + " Please retry!"); + console.log(response); + } + }, + fail: function (response) { + console.log('failed, ' + response); + showError("Something went wrong! Please reload!"); + } + }) +} + +$('.delete-btn').on("click", function () { + $('#delete-modal').modal('hide'); + if (deleteButtonId === "DELETE_QUEUE") { + deleteQueue(); + } else if (deleteButtonId === "DELETE_ALL") { + makeQueueEmpty(); + } else { + throw deleteButtonId; + } +}); +//======================================================================= \ No newline at end of file diff --git a/rqueue/src/main/resources/scripts/replace_message.lua b/rqueue/src/main/resources/scripts/move_message.lua similarity index 82% rename from rqueue/src/main/resources/scripts/replace_message.lua rename to rqueue/src/main/resources/scripts/move_message.lua index 1dad87f2..ac3a7cd6 100644 --- a/rqueue/src/main/resources/scripts/replace_message.lua +++ b/rqueue/src/main/resources/scripts/move_message.lua @@ -3,4 +3,5 @@ if score ~= nil then redis.call('ZADD', KEYS[2], ARGV[3], ARGV[2]) redis.call('ZREM', KEYS[1], ARGV[1]) end +score = tonumber(score) return score \ No newline at end of file diff --git a/rqueue/src/main/resources/templates/rqueue/base.html b/rqueue/src/main/resources/templates/rqueue/base.html index 7c0c62a6..5ca82059 100644 --- a/rqueue/src/main/resources/templates/rqueue/base.html +++ b/rqueue/src/main/resources/templates/rqueue/base.html @@ -92,7 +92,7 @@

Rqueue

Version: 

{{version}}
  • - Latest Version:  + Latest Version: 

    {{latestVersion}}
    • diff --git a/rqueue/src/test/java/com/github/sonus21/rqueue/config/RqueueConfigTest.java b/rqueue/src/test/java/com/github/sonus21/rqueue/config/RqueueConfigTest.java index bba2c3fb..09922248 100644 --- a/rqueue/src/test/java/com/github/sonus21/rqueue/config/RqueueConfigTest.java +++ b/rqueue/src/test/java/com/github/sonus21/rqueue/config/RqueueConfigTest.java @@ -65,7 +65,9 @@ public void getDelayedQueueName() { public void getDelayedQueueChannelName() { assertEquals("rqueue-channel::test", rqueueConfigVersion1.getDelayedQueueChannelName("test")); assertEquals("__rq::d-channel::test", rqueueConfigVersion2.getDelayedQueueChannelName("test")); - assertEquals("__rq::d-channel::{test}", rqueueConfigVersion2ClusterMode.getDelayedQueueChannelName("test")); + assertEquals( + "__rq::d-channel::{test}", + rqueueConfigVersion2ClusterMode.getDelayedQueueChannelName("test")); } @Test diff --git a/rqueue/src/test/java/com/github/sonus21/rqueue/config/RqueueListenerBaseConfigTest.java b/rqueue/src/test/java/com/github/sonus21/rqueue/config/RqueueListenerBaseConfigTest.java index f8f0576a..2f8c7aa3 100644 --- a/rqueue/src/test/java/com/github/sonus21/rqueue/config/RqueueListenerBaseConfigTest.java +++ b/rqueue/src/test/java/com/github/sonus21/rqueue/config/RqueueListenerBaseConfigTest.java @@ -59,16 +59,34 @@ public void testRqueueConfigSetConnectionFactoryFromBeanFactory() throws Illegal SimpleRqueueListenerContainerFactory factory = new SimpleRqueueListenerContainerFactory(); RqueueListenerConfig rqueueSystemConfig = createConfig(factory); doReturn(redisConnectionFactory).when(beanFactory).getBean(RedisConnectionFactory.class); - assertNotNull(rqueueSystemConfig.rqueueConfig(beanFactory, "__rq::version")); + assertNotNull(rqueueSystemConfig.rqueueConfig(beanFactory, "__rq::version", null)); assertNotNull(factory.getRedisConnectionFactory()); } + @Test + public void testRqueueConfigSetConnectionFactoryFromBeanFactoryWithDbVersion() + throws IllegalAccessException { + SimpleRqueueListenerContainerFactory factory = new SimpleRqueueListenerContainerFactory(); + RqueueListenerConfig rqueueSystemConfig = createConfig(factory); + doReturn(redisConnectionFactory).when(beanFactory).getBean(RedisConnectionFactory.class); + assertNotNull(rqueueSystemConfig.rqueueConfig(beanFactory, "__rq::version", 1)); + assertNotNull(factory.getRedisConnectionFactory()); + } + + @Test(expected = IllegalStateException.class) + public void testRqueueConfigInvalidDbVersion() throws IllegalAccessException { + SimpleRqueueListenerContainerFactory factory = new SimpleRqueueListenerContainerFactory(); + RqueueListenerConfig rqueueSystemConfig = createConfig(factory); + doReturn(redisConnectionFactory).when(beanFactory).getBean(RedisConnectionFactory.class); + rqueueSystemConfig.rqueueConfig(beanFactory, "__rq::version", 3); + } + @Test public void testRqueueConfigDoesNotChangeConnectionFactory() throws IllegalAccessException { SimpleRqueueListenerContainerFactory factory = new SimpleRqueueListenerContainerFactory(); factory.setRedisConnectionFactory(redisConnectionFactory); RqueueListenerConfig rqueueSystemConfig = createConfig(factory); - assertNotNull(rqueueSystemConfig.rqueueConfig(beanFactory, "__rq::version")); + assertNotNull(rqueueSystemConfig.rqueueConfig(beanFactory, "__rq::version", null)); assertNotNull(factory.getRedisConnectionFactory()); assertEquals(redisConnectionFactory, factory.getRedisConnectionFactory()); verify(beanFactory, times(0)).getBean(RedisConnectionFactory.class); diff --git a/rqueue/src/test/java/com/github/sonus21/rqueue/listener/RqueueExecutorTest.java b/rqueue/src/test/java/com/github/sonus21/rqueue/listener/RqueueExecutorTest.java index 3375ac18..d44f9be5 100644 --- a/rqueue/src/test/java/com/github/sonus21/rqueue/listener/RqueueExecutorTest.java +++ b/rqueue/src/test/java/com/github/sonus21/rqueue/listener/RqueueExecutorTest.java @@ -38,7 +38,6 @@ import com.github.sonus21.rqueue.web.service.RqueueMessageMetadataService; import java.lang.ref.WeakReference; import java.util.Collections; -import java.util.List; import java.util.UUID; import java.util.concurrent.Semaphore; import java.util.concurrent.atomic.AtomicInteger; @@ -47,6 +46,7 @@ import org.junit.runner.RunWith; import org.mockito.junit.MockitoJUnitRunner; import org.springframework.messaging.MessagingException; +import org.springframework.messaging.converter.CompositeMessageConverter; import org.springframework.messaging.converter.GenericMessageConverter; import org.springframework.messaging.converter.MessageConverter; @@ -65,13 +65,12 @@ public class RqueueExecutorTest { private RqueueMessageHandler messageHandler = mock(RqueueMessageHandler.class); private RqueueMessage rqueueMessage = new RqueueMessage(); private Semaphore semaphore = new Semaphore(100); + private MessageConverter messageConverter = new CompositeMessageConverter( Collections.singletonList(new GenericMessageConverter())); private int retryPerPoll = 3; private TaskExecutionBackOff taskBackOff = new FixedTaskExecutionBackOff(); @Before public void init() throws IllegalAccessException { - List messageConverterList = - Collections.singletonList(new GenericMessageConverter()); rqueueMessage.setMessage("test message"); rqueueMessage.setId(UUID.randomUUID().toString()); doReturn(rqueueWebConfig).when(container).getRqueueWebConfig(); @@ -80,7 +79,7 @@ public void init() throws IllegalAccessException { doReturn(discardProcessor).when(container).getDiscardMessageProcessor(); doReturn(true).when(container).isQueueActive(anyString()); doReturn(messageTemplate).when(container).getRqueueMessageTemplate(); - doReturn(messageConverterList).when(messageHandler).getMessageConverters(); + doReturn(messageConverter).when(messageHandler).getMessageConverter(); doReturn(preProcessMessageProcessor).when(container).getPreExecutionMessageProcessor(); doThrow(new MessagingException("Failing for some reason.")) .when(messageHandler) @@ -113,7 +112,7 @@ public void callDeadLetterProcessor() { semaphore, containerWeakReference, messageHandler, - 10, + -1, taskBackOff); rqueueExecutor.run(); assertEquals(1, deadLetterProcessor.getCount()); @@ -193,6 +192,33 @@ public void messageIsDeletedWhileExecuting() { verify(messageHandler, times(1)).handleMessage(any()); } + @Test + public void handleIgnoredMessage() { + QueueDetail queueDetail = TestUtils.createQueueDetail("test"); + MessageProcessor messageProcessor = + new MessageProcessor() { + @Override + public boolean process(Object message) { + return false; + } + }; + doReturn(messageProcessor).when(container).getPreExecutionMessageProcessor(); + + RqueueExecutor rqueueExecutor = + new RqueueExecutor( + rqueueMessage, + queueDetail, + semaphore, + containerWeakReference, + messageHandler, + 1, + taskBackOff); + rqueueExecutor.run(); + verify(messageHandler, times(0)).handleMessage(any()); + verify(messageTemplate, times(1)) + .removeElementFromZset(queueDetail.getProcessingQueueName(), rqueueMessage); + } + private class TestMessageProcessor implements MessageProcessor { private int count; diff --git a/rqueue/src/test/java/com/github/sonus21/rqueue/listener/RqueueMessageHandlerTest.java b/rqueue/src/test/java/com/github/sonus21/rqueue/listener/RqueueMessageHandlerTest.java index 70e743f4..61f192c7 100644 --- a/rqueue/src/test/java/com/github/sonus21/rqueue/listener/RqueueMessageHandlerTest.java +++ b/rqueue/src/test/java/com/github/sonus21/rqueue/listener/RqueueMessageHandlerTest.java @@ -23,7 +23,6 @@ import com.github.sonus21.rqueue.converter.GenericMessageConverter; import com.github.sonus21.rqueue.models.Concurrency; import com.github.sonus21.rqueue.utils.Constants; -import com.github.sonus21.rqueue.utils.MessageUtils; import java.lang.reflect.Method; import java.util.Collections; import java.util.HashMap; @@ -44,8 +43,9 @@ import org.springframework.core.env.MapPropertySource; import org.springframework.messaging.Message; import org.springframework.messaging.MessageHandler; +import org.springframework.messaging.MessageHeaders; import org.springframework.messaging.handler.annotation.MessageExceptionHandler; -import org.springframework.messaging.support.GenericMessage; +import org.springframework.messaging.support.MessageBuilder; @RunWith(MockitoJUnitRunner.StrictStubs.class) public class RqueueMessageHandlerTest { @@ -61,8 +61,9 @@ public class RqueueMessageHandlerTest { ((Message) messageConverter.toMessage(messagePayload, null)).getPayload(); private Message buildMessage(String queueName, String message) { - return new GenericMessage<>( - message, Collections.singletonMap(MessageUtils.getMessageHeaderKey(), queueName)); + return MessageBuilder.createMessage( + message, + new MessageHeaders(Collections.singletonMap(RqueueMessageHeaders.DESTINATION, queueName))); } @Test diff --git a/rqueue/src/test/java/com/github/sonus21/rqueue/listener/RqueueMessagePollerTest.java b/rqueue/src/test/java/com/github/sonus21/rqueue/listener/RqueueMessagePollerTest.java index 1ef5fc1b..b340e544 100644 --- a/rqueue/src/test/java/com/github/sonus21/rqueue/listener/RqueueMessagePollerTest.java +++ b/rqueue/src/test/java/com/github/sonus21/rqueue/listener/RqueueMessagePollerTest.java @@ -19,7 +19,6 @@ import com.github.sonus21.rqueue.utils.backoff.TaskExecutionBackOff; import java.util.HashMap; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; import org.junit.Test; public class RqueueMessagePollerTest { diff --git a/rqueue/src/test/java/com/github/sonus21/rqueue/utils/StringUtilsTest.java b/rqueue/src/test/java/com/github/sonus21/rqueue/utils/StringUtilsTest.java new file mode 100644 index 00000000..5208dc17 --- /dev/null +++ b/rqueue/src/test/java/com/github/sonus21/rqueue/utils/StringUtilsTest.java @@ -0,0 +1,81 @@ +/* + * Copyright 2020 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.sonus21.rqueue.utils; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +public class StringUtilsTest { + + @Test + public void isEmpty() { + assertTrue(StringUtils.isEmpty(null)); + assertTrue(StringUtils.isEmpty("")); + assertFalse(StringUtils.isEmpty("hello")); + } + + @Test + public void clean() { + assertEquals("", StringUtils.clean(" ")); + assertEquals("test", StringUtils.clean("test")); + assertEquals("test", StringUtils.clean(" test ")); + assertEquals("test queue", StringUtils.clean(" test queue ")); + } + + @Test(expected = IllegalArgumentException.class) + public void convertToCamelCaseEmpty() { + StringUtils.convertToCamelCase(" "); + } + + @Test(expected = IllegalArgumentException.class) + public void convertToCamelCaseNull() { + StringUtils.convertToCamelCase(null); + } + + @Test + public void convertToCamelCase() { + assertEquals("url", StringUtils.convertToCamelCase("URL")); + assertEquals("jobQueue", StringUtils.convertToCamelCase("job-Queue")); + assertEquals("jobQueue", StringUtils.convertToCamelCase("Job-Queue")); + assertEquals("jobQueue", StringUtils.convertToCamelCase("JOB-Queue")); + assertEquals("jobQueue", StringUtils.convertToCamelCase(" JOB-Queue ")); + assertEquals("jobQueue", StringUtils.convertToCamelCase(" JOB-1233-Queue ")); + assertEquals("jobQuEue", StringUtils.convertToCamelCase(" JOB-1233-Qu1234eue ")); + assertEquals("jobQuEueQueue", StringUtils.convertToCamelCase(" JOB-1233-Qu1234eueQUEUE ")); + assertEquals("1234-1234", StringUtils.convertToCamelCase(" 1234-1234 ")); + assertEquals("jB", StringUtils.convertToCamelCase(" jB ")); + assertEquals("jOb", StringUtils.convertToCamelCase(" jOB ")); + } + + @Test + public void groupName() { + assertEquals("Url", StringUtils.groupName("URL")); + assertEquals("JobQueue", StringUtils.groupName("job-Queue")); + assertEquals("JobQueue", StringUtils.groupName("Job-Queue")); + assertEquals("JobQueue", StringUtils.groupName("JOB-Queue")); + assertEquals("JobQueue", StringUtils.groupName(" JOB-Queue ")); + assertEquals("JobQueue", StringUtils.groupName(" JOB-1233-Queue ")); + assertEquals("JobQuEue", StringUtils.groupName(" JOB-1233-Qu1234eue ")); + assertEquals("JobQuEueQueue", StringUtils.groupName(" JOB-1233-Qu1234eueQUEUE ")); + assertEquals("1234-1234", StringUtils.groupName(" 1234-1234 ")); + assertEquals("JoB", StringUtils.groupName(" joB ")); + assertEquals("JB", StringUtils.groupName(" jB ")); + } +} diff --git a/rqueue/src/test/java/com/github/sonus21/rqueue/utils/backoff/FixedTaskExecutionBackOffTest.java b/rqueue/src/test/java/com/github/sonus21/rqueue/utils/backoff/FixedTaskExecutionBackOffTest.java index 3c96ddf9..c9e09bee 100644 --- a/rqueue/src/test/java/com/github/sonus21/rqueue/utils/backoff/FixedTaskExecutionBackOffTest.java +++ b/rqueue/src/test/java/com/github/sonus21/rqueue/utils/backoff/FixedTaskExecutionBackOffTest.java @@ -22,7 +22,6 @@ public class FixedTaskExecutionBackOffTest { - @Test public void construct() { FixedTaskExecutionBackOff backOff = new FixedTaskExecutionBackOff(1000, 100); diff --git a/rqueue/src/test/java/com/github/sonus21/rqueue/web/service/RqueueQDetailServiceTest.java b/rqueue/src/test/java/com/github/sonus21/rqueue/web/service/RqueueQDetailServiceTest.java index adeec712..a5221e92 100644 --- a/rqueue/src/test/java/com/github/sonus21/rqueue/web/service/RqueueQDetailServiceTest.java +++ b/rqueue/src/test/java/com/github/sonus21/rqueue/web/service/RqueueQDetailServiceTest.java @@ -395,7 +395,7 @@ public void getScheduledTasks() { doReturn(Arrays.asList(queueConfig, queueConfig2)) .when(rqueueSystemManagerService) - .getQueueConfigs(); + .getSortedQueueConfigs(); doReturn(newArrayList(100L, 200L)) .when(redisTemplate) @@ -415,7 +415,7 @@ public void getScheduledTasks() { @Test public void getWaitingTasks() { doReturn(redisTemplate).when(stringRqueueRedisTemplate).getRedisTemplate(); - doReturn(queueConfigList).when(rqueueSystemManagerService).getQueueConfigs(); + doReturn(queueConfigList).when(rqueueSystemManagerService).getSortedQueueConfigs(); doReturn(Arrays.asList(100L, 110L)) .when(redisTemplate) .executePipelined(any(RedisCallback.class)); @@ -430,7 +430,7 @@ public void getWaitingTasks() { @Test public void getRunningTasks() { doReturn(redisTemplate).when(stringRqueueRedisTemplate).getRedisTemplate(); - doReturn(queueConfigList).when(rqueueSystemManagerService).getQueueConfigs(); + doReturn(queueConfigList).when(rqueueSystemManagerService).getSortedQueueConfigs(); doReturn(Arrays.asList(100L, 110L)) .when(redisTemplate) .executePipelined(any(RedisCallback.class)); @@ -447,7 +447,7 @@ public void getRunningTasks() { @Test public void getDeadLetterTasks() { doReturn(redisTemplate).when(stringRqueueRedisTemplate).getRedisTemplate(); - doReturn(queueConfigList).when(rqueueSystemManagerService).getQueueConfigs(); + doReturn(queueConfigList).when(rqueueSystemManagerService).getSortedQueueConfigs(); doReturn(Arrays.asList(100L, 110L)) .when(redisTemplate) .executePipelined(any(RedisCallback.class)); diff --git a/rqueue/src/test/java/com/github/sonus21/rqueue/web/service/RqueueSystemManagerServiceTest.java b/rqueue/src/test/java/com/github/sonus21/rqueue/web/service/RqueueSystemManagerServiceTest.java index f54095c7..2b805542 100644 --- a/rqueue/src/test/java/com/github/sonus21/rqueue/web/service/RqueueSystemManagerServiceTest.java +++ b/rqueue/src/test/java/com/github/sonus21/rqueue/web/service/RqueueSystemManagerServiceTest.java @@ -116,6 +116,21 @@ public void getQueueConfigs() { rqueueSystemManagerService.getQueueConfigs()); } + @Test + public void getSortedQueueConfigs() { + doReturn(queues).when(stringRqueueRedisTemplate).getMembers(TestUtils.getQueuesKey()); + doReturn(Arrays.asList(slowQueueConfig, fastQueueConfig)) + .when(rqueueSystemConfigDao) + .findAllQConfig( + queues.stream() + .map(TestUtils::getQueueConfigKey) + .sorted() + .collect(Collectors.toList())); + assertEquals( + Arrays.asList(fastQueueConfig, slowQueueConfig), + rqueueSystemManagerService.getSortedQueueConfigs()); + } + @Test public void testGetQueueConfigs() { doReturn(Arrays.asList(slowQueueConfig, fastQueueConfig))