From 0ea9d7fa19185c34b38f04ecf2a40cd265483851 Mon Sep 17 00:00:00 2001 From: Sonu Kumar Date: Sun, 15 Nov 2020 21:47:48 +0530 Subject: [PATCH 01/23] Generic message converter --- build.gradle | 8 +- .../annotation/MessageGenericField.java | 28 +++++++ .../converter/GenericMessageConverter.java | 77 ++++++++++++++----- 3 files changed, 90 insertions(+), 23 deletions(-) create mode 100644 rqueue-core/src/main/java/com/github/sonus21/rqueue/annotation/MessageGenericField.java diff --git a/build.gradle b/build.gradle index 86c092e7..17814d01 100644 --- a/build.gradle +++ b/build.gradle @@ -26,10 +26,10 @@ ext { springDataVersion = System.getenv("SPRING_DATA_VERSION") microMeterVersion = System.getenv("MICROMETER_VERSION") -// springBootVersion = '2.1.0.RELEASE' -// springVersion = '5.1.2.RELEASE' -// springDataVersion = '2.1.2.RELEASE' -// microMeterVersion = '1.1.0' + springBootVersion = '2.1.0.RELEASE' + springVersion = '5.1.2.RELEASE' + springDataVersion = '2.1.2.RELEASE' + microMeterVersion = '1.1.0' // logging dependencies lombokVersion = '1.18.10' diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/annotation/MessageGenericField.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/annotation/MessageGenericField.java new file mode 100644 index 00000000..155f589a --- /dev/null +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/annotation/MessageGenericField.java @@ -0,0 +1,28 @@ +/* + * Copyright 2020 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.sonus21.rqueue.annotation; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target(ElementType.FIELD) +@Retention(RetentionPolicy.RUNTIME) +@Documented +public @interface MessageGenericField {} diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/GenericMessageConverter.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/GenericMessageConverter.java index ecc2f917..977b6409 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/GenericMessageConverter.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/GenericMessageConverter.java @@ -16,11 +16,16 @@ package com.github.sonus21.rqueue.converter; +import static org.springframework.util.Assert.notNull; + import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JavaType; import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.sonus21.rqueue.annotation.MessageGenericField; import com.github.sonus21.rqueue.utils.SerializationUtils; +import java.lang.reflect.Field; import java.util.Collection; +import java.util.LinkedList; import java.util.List; import lombok.AllArgsConstructor; import lombok.Getter; @@ -37,9 +42,17 @@ * vice versa. */ @Slf4j -@SuppressWarnings("unchecked") public class GenericMessageConverter implements MessageConverter { - private static final ObjectMapper objectMapper = new ObjectMapper(); + private final ObjectMapper objectMapper; + + public GenericMessageConverter() { + this.objectMapper = new ObjectMapper(); + } + + public GenericMessageConverter(ObjectMapper objectMapper) { + notNull(objectMapper, "objectMapper cannot be null"); + this.objectMapper = objectMapper; + } /** * Convert the payload of a {@link Message} from a serialized form to a typed Object of type @@ -65,12 +78,12 @@ public Object fromMessage(Message message, Class targetClass) { } Class envelopeClass = Thread.currentThread().getContextClassLoader().loadClass(classNames[0]); - Class elementClass = - Thread.currentThread().getContextClassLoader().loadClass(classNames[1]); + Class[] classes = new Class[classNames.length - 1]; + for (int i = 1; i < classNames.length; i++) { + classes[i - 1] = Thread.currentThread().getContextClassLoader().loadClass(classNames[i]); + } JavaType type = - objectMapper - .getTypeFactory() - .constructCollectionType((Class) envelopeClass, elementClass); + objectMapper.getTypeFactory().constructParametricType(envelopeClass, classes); return objectMapper.readValue(msg.msg, type); } } catch (Exception e) { @@ -83,8 +96,41 @@ private String[] splitClassNames(String name) { return name.split("#"); } - private String getClassName(String name, List payload) { - return name + '#' + payload.get(0).getClass().getName(); + private String getClassNameForCollection(String name, Collection payload) { + if (payload instanceof List) { + if (payload.isEmpty()) { + return null; + } + return name + '#' + ((List) payload).get(0).getClass().getName(); + } + return null; + } + + private String getGenericFieldBasedClassName(String name, Object payload) { + List genericFieldClassNames = new LinkedList<>(); + for (Field field : payload.getClass().getDeclaredFields()) { + if (field.isAnnotationPresent(MessageGenericField.class)) { + try { + Object fieldVal = field.get(payload); + genericFieldClassNames.add(fieldVal.getClass().getName()); + } catch (IllegalAccessException e) { + log.error("Field can not be read", e); + return null; + } + } + } + if (genericFieldClassNames.isEmpty()) { + return name; + } + return name + '#' + String.join("#", genericFieldClassNames); + } + + private String getClassName(Object payload) { + String name = payload.getClass().getName(); + if (payload instanceof Collection) { + return getClassNameForCollection(name, (Collection) payload); + } + return getGenericFieldBasedClassName(name, payload); } /** @@ -99,16 +145,9 @@ private String getClassName(String name, List payload) { */ @Override public Message toMessage(Object payload, MessageHeaders headers) { - String name = payload.getClass().getName(); - if (payload instanceof Collection) { - if (payload instanceof List) { - if (((List) payload).isEmpty()) { - return null; - } - name = getClassName(name, (List) payload); - } else { - return null; - } + String name = getClassName(payload); + if (name == null) { + return null; } try { String msg = objectMapper.writeValueAsString(payload); From 862d428f9e66d3006545785b8528757d74c031d6 Mon Sep 17 00:00:00 2001 From: Sonu Kumar Date: Mon, 16 Nov 2020 19:34:04 +0530 Subject: [PATCH 02/23] Generic class one inside other. --- .../converter/GenericMessageConverter.java | 34 ++++-- .../GenericMessageConverterTest.java | 106 +++++++++++++++++- 2 files changed, 128 insertions(+), 12 deletions(-) diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/GenericMessageConverter.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/GenericMessageConverter.java index 977b6409..9e8082f7 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/GenericMessageConverter.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/GenericMessageConverter.java @@ -21,7 +21,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JavaType; import com.fasterxml.jackson.databind.ObjectMapper; -import com.github.sonus21.rqueue.annotation.MessageGenericField; import com.github.sonus21.rqueue.utils.SerializationUtils; import java.lang.reflect.Field; import java.util.Collection; @@ -32,6 +31,7 @@ import lombok.NoArgsConstructor; import lombok.Setter; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.reflect.FieldUtils; import org.springframework.messaging.Message; import org.springframework.messaging.MessageHeaders; import org.springframework.messaging.converter.MessageConverter; @@ -106,18 +106,30 @@ private String getClassNameForCollection(String name, Collection payload) { return null; } - private String getGenericFieldBasedClassName(String name, Object payload) { - List genericFieldClassNames = new LinkedList<>(); + private void genericFieldClassNames(List genericFieldClassNames, Object payload) + throws IllegalAccessException { for (Field field : payload.getClass().getDeclaredFields()) { - if (field.isAnnotationPresent(MessageGenericField.class)) { - try { - Object fieldVal = field.get(payload); - genericFieldClassNames.add(fieldVal.getClass().getName()); - } catch (IllegalAccessException e) { - log.error("Field can not be read", e); - return null; - } + if (field.getGenericType().equals(field.getType())) { + continue; } + Object fieldVal = FieldUtils.readField(field, payload, true); + String genericFieldType = field.getGenericType().getTypeName(); + Class fieldClass = fieldVal.getClass(); + if (genericFieldType.endsWith(">")) { + genericFieldClassNames(genericFieldClassNames, fieldVal); + } else { + genericFieldClassNames.add(fieldClass.getName()); + } + } + } + + private String getGenericFieldBasedClassName(String name, Object payload) { + List genericFieldClassNames = new LinkedList<>(); + try { + genericFieldClassNames(genericFieldClassNames, payload); + } catch (IllegalAccessException e) { + log.error("Field can not be read", e); + return null; } if (genericFieldClassNames.isEmpty()) { return name; diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/converter/GenericMessageConverterTest.java b/rqueue-core/src/test/java/com/github/sonus21/rqueue/converter/GenericMessageConverterTest.java index 3bb1328a..8531f61d 100644 --- a/rqueue-core/src/test/java/com/github/sonus21/rqueue/converter/GenericMessageConverterTest.java +++ b/rqueue-core/src/test/java/com/github/sonus21/rqueue/converter/GenericMessageConverterTest.java @@ -19,8 +19,11 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; +import com.github.sonus21.rqueue.annotation.MessageGenericField; import com.github.sonus21.rqueue.listener.RqueueMessageHeaders; +import java.util.Arrays; import java.util.Collections; +import java.util.List; import java.util.Objects; import java.util.UUID; import lombok.AllArgsConstructor; @@ -84,10 +87,111 @@ public void toMessageEmptyList() { Collections.emptyList(), RqueueMessageHeaders.emptyMessageHeaders())); } + @Test + public void testMessageNonEmptyList() { + assertNull( + genericMessageConverter.toMessage( + Collections.emptyList(), RqueueMessageHeaders.emptyMessageHeaders())); + } + + @Test + public void testToAndFromMessageList() { + List dataList = Arrays.asList(testData); + Message message = + genericMessageConverter.toMessage(dataList, RqueueMessageHeaders.emptyMessageHeaders()); + List fromMessage = + (List) genericMessageConverter.fromMessage(message, null); + assertEquals(dataList, fromMessage); + } + + @Test + public void testGenericMessageToAndFrom() { + GenericTestData data = new GenericTestData<>(10, testData); + Message message = + genericMessageConverter.toMessage(data, RqueueMessageHeaders.emptyMessageHeaders()); + GenericTestData fromMessage = + (GenericTestData) genericMessageConverter.fromMessage(message, null); + assertEquals(data, fromMessage); + } + + @Test + public void testMultipleGenericFieldMessageToAndFrom() { + MultiGenericTestData data = new MultiGenericTestData<>("Test", 10, testData); + Message message = + genericMessageConverter.toMessage(data, RqueueMessageHeaders.emptyMessageHeaders()); + MultiGenericTestData fromMessage = + (MultiGenericTestData) genericMessageConverter.fromMessage(message, null); + assertEquals(data, fromMessage); + } + + @Test + public void testMultiLevelGenericMessageToAndFrom() { + GenericTestData testData = new GenericTestData<>(10, "foo"); + GenericTestData testData2 = new GenericTestData<>(100, 200); + MultiLevelGenericTestData data = + new MultiLevelGenericTestData<>("test", testData, testData2); + Message message = + genericMessageConverter.toMessage(data, RqueueMessageHeaders.emptyMessageHeaders()); + MultiLevelGenericTestData fromMessage = + (MultiLevelGenericTestData) + genericMessageConverter.fromMessage(message, null); + assertEquals(data, fromMessage); + } + + @Test + public void testMultiLevelGenericMessageToAndFromWithoutAllArgsConstructor() { + GenericTestData testData = new GenericTestData<>(10, "foo"); + GenericTestData testData2 = new GenericTestData<>(100, 200); + MultiLevelGenericTestDataNoArgs data = new MultiLevelGenericTestDataNoArgs<>(); + data.setData("Test"); + data.setTGenericTestData(testData); + data.setVGenericTestData(testData2); + Message message = + genericMessageConverter.toMessage(data, RqueueMessageHeaders.emptyMessageHeaders()); + MultiLevelGenericTestDataNoArgs fromMessage = + (MultiLevelGenericTestDataNoArgs) + genericMessageConverter.fromMessage(message, null); + assertEquals(data, fromMessage); + } + + @Data + @NoArgsConstructor + public static class MultiLevelGenericTestDataNoArgs { + private String data; + @MessageGenericField private GenericTestData tGenericTestData; + @MessageGenericField private GenericTestData vGenericTestData; + } + + @Data + @AllArgsConstructor + @NoArgsConstructor + public static class MultiLevelGenericTestData { + private String data; + @MessageGenericField private GenericTestData tGenericTestData; + @MessageGenericField private GenericTestData vGenericTestData; + } + + @Data + @AllArgsConstructor + @NoArgsConstructor + public static class MultiGenericTestData { + @MessageGenericField private K key; + @MessageGenericField private V value; + private TestData testData; + } + + @Data + @AllArgsConstructor + @NoArgsConstructor + public static class GenericTestData { + private Integer index; + @MessageGenericField private T data; + } + @Data @AllArgsConstructor @NoArgsConstructor - private static class TestData { + public static class TestData { private String id; private String message; } From 6a4affc8e9f4d08a3b2d2e0e2f43d867913c3d0b Mon Sep 17 00:00:00 2001 From: Sonu Kumar Date: Mon, 16 Nov 2020 19:36:25 +0530 Subject: [PATCH 03/23] removed unused annotation --- .../annotation/MessageGenericField.java | 28 ------------------- .../GenericMessageConverterTest.java | 15 +++++----- 2 files changed, 7 insertions(+), 36 deletions(-) delete mode 100644 rqueue-core/src/main/java/com/github/sonus21/rqueue/annotation/MessageGenericField.java diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/annotation/MessageGenericField.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/annotation/MessageGenericField.java deleted file mode 100644 index 155f589a..00000000 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/annotation/MessageGenericField.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2020 Sonu Kumar - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.github.sonus21.rqueue.annotation; - -import java.lang.annotation.Documented; -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -@Target(ElementType.FIELD) -@Retention(RetentionPolicy.RUNTIME) -@Documented -public @interface MessageGenericField {} diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/converter/GenericMessageConverterTest.java b/rqueue-core/src/test/java/com/github/sonus21/rqueue/converter/GenericMessageConverterTest.java index 8531f61d..9289f074 100644 --- a/rqueue-core/src/test/java/com/github/sonus21/rqueue/converter/GenericMessageConverterTest.java +++ b/rqueue-core/src/test/java/com/github/sonus21/rqueue/converter/GenericMessageConverterTest.java @@ -19,7 +19,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; -import com.github.sonus21.rqueue.annotation.MessageGenericField; import com.github.sonus21.rqueue.listener.RqueueMessageHeaders; import java.util.Arrays; import java.util.Collections; @@ -158,8 +157,8 @@ public void testMultiLevelGenericMessageToAndFromWithoutAllArgsConstructor() { @NoArgsConstructor public static class MultiLevelGenericTestDataNoArgs { private String data; - @MessageGenericField private GenericTestData tGenericTestData; - @MessageGenericField private GenericTestData vGenericTestData; + private GenericTestData tGenericTestData; + private GenericTestData vGenericTestData; } @Data @@ -167,16 +166,16 @@ public static class MultiLevelGenericTestDataNoArgs { @NoArgsConstructor public static class MultiLevelGenericTestData { private String data; - @MessageGenericField private GenericTestData tGenericTestData; - @MessageGenericField private GenericTestData vGenericTestData; + private GenericTestData tGenericTestData; + private GenericTestData vGenericTestData; } @Data @AllArgsConstructor @NoArgsConstructor public static class MultiGenericTestData { - @MessageGenericField private K key; - @MessageGenericField private V value; + private K key; + private V value; private TestData testData; } @@ -185,7 +184,7 @@ public static class MultiGenericTestData { @NoArgsConstructor public static class GenericTestData { private Integer index; - @MessageGenericField private T data; + private T data; } @Data From f496defe7f805e0e5e320975e0dc182c1afba2f5 Mon Sep 17 00:00:00 2001 From: Sonu Kumar Date: Mon, 16 Nov 2020 20:43:03 +0530 Subject: [PATCH 04/23] null ptr issue --- .../rqueue/converter/GenericMessageConverter.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/GenericMessageConverter.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/GenericMessageConverter.java index 9e8082f7..0ff5d44c 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/GenericMessageConverter.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/GenericMessageConverter.java @@ -106,19 +106,22 @@ private String getClassNameForCollection(String name, Collection payload) { return null; } - private void genericFieldClassNames(List genericFieldClassNames, Object payload) + private void genericFieldClassNames(List classNames, Object payload) throws IllegalAccessException { for (Field field : payload.getClass().getDeclaredFields()) { if (field.getGenericType().equals(field.getType())) { continue; } Object fieldVal = FieldUtils.readField(field, payload, true); + if (fieldVal == null) { + continue; + } String genericFieldType = field.getGenericType().getTypeName(); Class fieldClass = fieldVal.getClass(); if (genericFieldType.endsWith(">")) { - genericFieldClassNames(genericFieldClassNames, fieldVal); + genericFieldClassNames(classNames, fieldVal); } else { - genericFieldClassNames.add(fieldClass.getName()); + classNames.add(fieldClass.getName()); } } } From 49960b1330eef350225a6c8a453f22d7e27cbb33 Mon Sep 17 00:00:00 2001 From: Sonu Kumar Date: Mon, 16 Nov 2020 20:51:04 +0530 Subject: [PATCH 05/23] uncomment gradle properties --- build.gradle | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build.gradle b/build.gradle index 17814d01..86c092e7 100644 --- a/build.gradle +++ b/build.gradle @@ -26,10 +26,10 @@ ext { springDataVersion = System.getenv("SPRING_DATA_VERSION") microMeterVersion = System.getenv("MICROMETER_VERSION") - springBootVersion = '2.1.0.RELEASE' - springVersion = '5.1.2.RELEASE' - springDataVersion = '2.1.2.RELEASE' - microMeterVersion = '1.1.0' +// springBootVersion = '2.1.0.RELEASE' +// springVersion = '5.1.2.RELEASE' +// springDataVersion = '2.1.2.RELEASE' +// microMeterVersion = '1.1.0' // logging dependencies lombokVersion = '1.18.10' From 0dadc59a65c0c7cbc79cb2bbf673e9c20cfdad26 Mon Sep 17 00:00:00 2001 From: Sonu Kumar Date: Sat, 28 Nov 2020 20:12:19 +0530 Subject: [PATCH 06/23] Do not support generic type --- .../converter/GenericMessageConverter.java | 58 ++-- .../converter/RqueueRedisSerializer.java | 15 +- .../sonus21/rqueue/utils/RedisUtils.java | 32 ++- .../GenericMessageConverterTest.java | 249 +++++++++++++++--- 4 files changed, 266 insertions(+), 88 deletions(-) diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/GenericMessageConverter.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/GenericMessageConverter.java index 0ff5d44c..4215f89b 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/GenericMessageConverter.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/GenericMessageConverter.java @@ -22,16 +22,14 @@ import com.fasterxml.jackson.databind.JavaType; import com.fasterxml.jackson.databind.ObjectMapper; import com.github.sonus21.rqueue.utils.SerializationUtils; -import java.lang.reflect.Field; +import java.lang.reflect.TypeVariable; import java.util.Collection; -import java.util.LinkedList; import java.util.List; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.reflect.FieldUtils; import org.springframework.messaging.Message; import org.springframework.messaging.MessageHeaders; import org.springframework.messaging.converter.MessageConverter; @@ -44,14 +42,24 @@ @Slf4j public class GenericMessageConverter implements MessageConverter { private final ObjectMapper objectMapper; + private final boolean genericFieldEnabled; public GenericMessageConverter() { - this.objectMapper = new ObjectMapper(); + this(new ObjectMapper()); } public GenericMessageConverter(ObjectMapper objectMapper) { + this(objectMapper, true); + } + + public GenericMessageConverter(boolean genericFieldEnabled) { + this(new ObjectMapper(), genericFieldEnabled); + } + + public GenericMessageConverter(ObjectMapper objectMapper, boolean genericFieldEnabled) { notNull(objectMapper, "objectMapper cannot be null"); this.objectMapper = objectMapper; + this.genericFieldEnabled = genericFieldEnabled; } /** @@ -106,46 +114,24 @@ private String getClassNameForCollection(String name, Collection payload) { return null; } - private void genericFieldClassNames(List classNames, Object payload) - throws IllegalAccessException { - for (Field field : payload.getClass().getDeclaredFields()) { - if (field.getGenericType().equals(field.getType())) { - continue; - } - Object fieldVal = FieldUtils.readField(field, payload, true); - if (fieldVal == null) { - continue; - } - String genericFieldType = field.getGenericType().getTypeName(); - Class fieldClass = fieldVal.getClass(); - if (genericFieldType.endsWith(">")) { - genericFieldClassNames(classNames, fieldVal); - } else { - classNames.add(fieldClass.getName()); - } + private String getGenericFieldBasedClassName(Class clazz, Object payload) { + if (!genericFieldEnabled) { + return clazz.getName(); } - } - - private String getGenericFieldBasedClassName(String name, Object payload) { - List genericFieldClassNames = new LinkedList<>(); - try { - genericFieldClassNames(genericFieldClassNames, payload); - } catch (IllegalAccessException e) { - log.error("Field can not be read", e); - return null; - } - if (genericFieldClassNames.isEmpty()) { - return name; + TypeVariable[] typeVariables = clazz.getTypeParameters(); + if (typeVariables.length == 0) { + return clazz.getName(); } - return name + '#' + String.join("#", genericFieldClassNames); + return null; } private String getClassName(Object payload) { - String name = payload.getClass().getName(); + Class payloadClass = payload.getClass(); + String name = payloadClass.getName(); if (payload instanceof Collection) { return getClassNameForCollection(name, (Collection) payload); } - return getGenericFieldBasedClassName(name, payload); + return getGenericFieldBasedClassName(payloadClass, payload); } /** diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/RqueueRedisSerializer.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/RqueueRedisSerializer.java index b9a62c1a..509cbbb9 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/RqueueRedisSerializer.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/RqueueRedisSerializer.java @@ -8,12 +8,19 @@ @Slf4j public class RqueueRedisSerializer implements RedisSerializer { - private GenericJackson2JsonRedisSerializer jackson2JsonRedisSerializer = - new GenericJackson2JsonRedisSerializer(); + private final RedisSerializer serializer; + + public RqueueRedisSerializer(RedisSerializer redisSerializer) { + this.serializer = redisSerializer; + } + + public RqueueRedisSerializer() { + this(new GenericJackson2JsonRedisSerializer()); + } @Override public byte[] serialize(Object t) throws SerializationException { - return jackson2JsonRedisSerializer.serialize(t); + return serializer.serialize(t); } @Override @@ -22,7 +29,7 @@ public Object deserialize(byte[] bytes) throws SerializationException { return null; } try { - return jackson2JsonRedisSerializer.deserialize(bytes); + return serializer.deserialize(bytes); } catch (Exception e) { log.warn("Jackson deserialization has failed {}", new String(bytes), e); return new String(bytes); diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/RedisUtils.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/RedisUtils.java index 37b98082..582637f4 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/RedisUtils.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/RedisUtils.java @@ -29,17 +29,31 @@ public class RedisUtils { private RedisUtils() {} + public interface RedisTemplateProvider { + RedisTemplate getRedisTemplate(RedisConnectionFactory redisConnectionFactory); + } + + @SuppressWarnings({"java:S1104","java:S1444"}) + public static RedisTemplateProvider redisTemplateProvider = + new RedisTemplateProvider() { + @Override + public RedisTemplate getRedisTemplate( + RedisConnectionFactory redisConnectionFactory) { + RedisTemplate redisTemplate = new RedisTemplate<>(); + StringRedisSerializer stringRedisSerializer = new StringRedisSerializer(); + RqueueRedisSerializer rqueueRedisSerializer = new RqueueRedisSerializer(); + redisTemplate.setConnectionFactory(redisConnectionFactory); + redisTemplate.setKeySerializer(stringRedisSerializer); + redisTemplate.setValueSerializer(rqueueRedisSerializer); + redisTemplate.setHashKeySerializer(stringRedisSerializer); + redisTemplate.setHashValueSerializer(rqueueRedisSerializer); + return redisTemplate; + } + }; + public static RedisTemplate getRedisTemplate( RedisConnectionFactory redisConnectionFactory) { - RedisTemplate redisTemplate = new RedisTemplate<>(); - StringRedisSerializer stringRedisSerializer = new StringRedisSerializer(); - RqueueRedisSerializer rqueueRedisSerializer = new RqueueRedisSerializer(); - redisTemplate.setConnectionFactory(redisConnectionFactory); - redisTemplate.setKeySerializer(stringRedisSerializer); - redisTemplate.setValueSerializer(rqueueRedisSerializer); - redisTemplate.setHashKeySerializer(stringRedisSerializer); - redisTemplate.setHashValueSerializer(rqueueRedisSerializer); - return redisTemplate; + return redisTemplateProvider.getRedisTemplate(redisConnectionFactory); } public static List executePipeLine( diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/converter/GenericMessageConverterTest.java b/rqueue-core/src/test/java/com/github/sonus21/rqueue/converter/GenericMessageConverterTest.java index 9289f074..8976f660 100644 --- a/rqueue-core/src/test/java/com/github/sonus21/rqueue/converter/GenericMessageConverterTest.java +++ b/rqueue-core/src/test/java/com/github/sonus21/rqueue/converter/GenericMessageConverterTest.java @@ -20,14 +20,21 @@ import static org.junit.jupiter.api.Assertions.assertNull; import com.github.sonus21.rqueue.listener.RqueueMessageHeaders; +import java.lang.reflect.GenericArrayType; +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; +import java.lang.reflect.TypeVariable; +import java.lang.reflect.WildcardType; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.UUID; +import java.util.stream.Collectors; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.junit.jupiter.MockitoExtension; @@ -35,9 +42,12 @@ import org.springframework.messaging.support.GenericMessage; @ExtendWith(MockitoExtension.class) +@SuppressWarnings("unchecked") public class GenericMessageConverterTest { - private GenericMessageConverter genericMessageConverter = new GenericMessageConverter(); - private TestData testData = new TestData(UUID.randomUUID().toString(), "This is test"); + private static final GenericMessageConverter genericMessageConverter = + new GenericMessageConverter(); + private static Comment comment = new Comment(UUID.randomUUID().toString(), "This is test"); + private static Email email = new Email(UUID.randomUUID().toString(), "This is test"); @Test public void fromMessageIoException() { @@ -47,16 +57,16 @@ public void fromMessageIoException() { @Test public void fromMessageClassCastException() { - Message message1 = new GenericMessage<>(testData); - assertNull(genericMessageConverter.fromMessage(message1, null)); + Message commentMessage = new GenericMessage<>(comment); + assertNull(genericMessageConverter.fromMessage(commentMessage, null)); } @Test public void fromMessageClassNotFoundException() { - Message message2 = (Message) genericMessageConverter.toMessage(testData, null); - String payload = Objects.requireNonNull(message2).getPayload().replace("TestData", "SomeData"); - Message message3 = new GenericMessage<>(payload); - assertNull(genericMessageConverter.fromMessage(message3, null)); + Message message = (Message) genericMessageConverter.toMessage(comment, null); + String payload = Objects.requireNonNull(message).getPayload().replace("Comment", "SomeData"); + Message updatedMessage = new GenericMessage<>(payload); + assertNull(genericMessageConverter.fromMessage(updatedMessage, null)); } @Test @@ -67,9 +77,9 @@ public void toMessageEmptyObject() { @Test public void toMessage() { - Message m = (Message) genericMessageConverter.toMessage(testData, null); - TestData t2 = (TestData) genericMessageConverter.fromMessage(m, null); - assertEquals(testData, t2); + Message m = (Message) genericMessageConverter.toMessage(comment, null); + Comment comment1 = (Comment) genericMessageConverter.fromMessage(m, null); + assertEquals(comment1, comment); } @Test @@ -95,60 +105,116 @@ public void testMessageNonEmptyList() { @Test public void testToAndFromMessageList() { - List dataList = Arrays.asList(testData); - Message message = + List dataList = Collections.singletonList(comment); + Message message = genericMessageConverter.toMessage(dataList, RqueueMessageHeaders.emptyMessageHeaders()); - List fromMessage = - (List) genericMessageConverter.fromMessage(message, null); + List fromMessage = (List) genericMessageConverter.fromMessage(message, null); assertEquals(dataList, fromMessage); } @Test - public void testGenericMessageToAndFrom() { - GenericTestData data = new GenericTestData<>(10, testData); - Message message = + public void testGenericMessageToReturnNull() { + GenericTestData data = new GenericTestData<>(10, comment); + Message message = genericMessageConverter.toMessage(data, RqueueMessageHeaders.emptyMessageHeaders()); - GenericTestData fromMessage = - (GenericTestData) genericMessageConverter.fromMessage(message, null); - assertEquals(data, fromMessage); + assertNull(message); } @Test + @Disabled public void testMultipleGenericFieldMessageToAndFrom() { - MultiGenericTestData data = new MultiGenericTestData<>("Test", 10, testData); - Message message = + MultiGenericTestData data = new MultiGenericTestData<>(10, comment, email); + Message message = + genericMessageConverter.toMessage(data, RqueueMessageHeaders.emptyMessageHeaders()); + MultiGenericTestData fromMessage = + (MultiGenericTestData) genericMessageConverter.fromMessage(message, null); + assertEquals(data, fromMessage); + } + + @Test + @Disabled + public void testMultipleGenericSameTypeMessageToAndFrom() { + MultiGenericTestData data = new MultiGenericTestData<>(10, comment, comment); + Message message = genericMessageConverter.toMessage(data, RqueueMessageHeaders.emptyMessageHeaders()); - MultiGenericTestData fromMessage = - (MultiGenericTestData) genericMessageConverter.fromMessage(message, null); + MultiGenericTestData fromMessage = + (MultiGenericTestData) genericMessageConverter.fromMessage(message, null); assertEquals(data, fromMessage); } @Test + @Disabled public void testMultiLevelGenericMessageToAndFrom() { - GenericTestData testData = new GenericTestData<>(10, "foo"); - GenericTestData testData2 = new GenericTestData<>(100, 200); - MultiLevelGenericTestData data = + GenericTestData testData = new GenericTestData<>(10, comment); + GenericTestData testData2 = new GenericTestData<>(100, email); + MultiLevelGenericTestData data = new MultiLevelGenericTestData<>("test", testData, testData2); - Message message = + Message message = genericMessageConverter.toMessage(data, RqueueMessageHeaders.emptyMessageHeaders()); - MultiLevelGenericTestData fromMessage = - (MultiLevelGenericTestData) + MultiLevelGenericTestData fromMessage = + (MultiLevelGenericTestData) genericMessageConverter.fromMessage(message, null); assertEquals(data, fromMessage); } @Test + @Disabled public void testMultiLevelGenericMessageToAndFromWithoutAllArgsConstructor() { - GenericTestData testData = new GenericTestData<>(10, "foo"); - GenericTestData testData2 = new GenericTestData<>(100, 200); - MultiLevelGenericTestDataNoArgs data = new MultiLevelGenericTestDataNoArgs<>(); + GenericTestData testData = new GenericTestData<>(10, comment); + GenericTestData testData2 = new GenericTestData<>(100, email); + MultiLevelGenericTestDataNoArgs data = new MultiLevelGenericTestDataNoArgs<>(); data.setData("Test"); data.setTGenericTestData(testData); data.setVGenericTestData(testData2); - Message message = + Message message = + genericMessageConverter.toMessage(data, RqueueMessageHeaders.emptyMessageHeaders()); + MultiLevelGenericTestDataNoArgs fromMessage = + (MultiLevelGenericTestDataNoArgs) + genericMessageConverter.fromMessage(message, null); + assertEquals(data, fromMessage); + } + + @Test + @Disabled + public void testPredefinedGenericTypeToFromMessage() { + MultiGenericTestData multiGenericTestData = + new MultiGenericTestData<>(10, comment, email); + GenericTestDataWithPredefinedType data = + new GenericTestDataWithPredefinedType(200, multiGenericTestData); + Message message = + genericMessageConverter.toMessage(data, RqueueMessageHeaders.emptyMessageHeaders()); + GenericTestDataWithPredefinedType fromMessage = + (GenericTestDataWithPredefinedType) genericMessageConverter.fromMessage(message, null); + assertEquals(data, fromMessage); + } + + @Test + @Disabled + public void testMultiGenericSameTypeToFromMessage() { + GenericTestData genericTestData = new GenericTestData<>(100, comment); + MultiGenericTestData multiGenericTestData = + new MultiGenericTestData<>(200, comment, comment); + MultiGenericTestDataSameType data = + new MultiGenericTestDataSameType<>(10, genericTestData, multiGenericTestData); + Message message = + genericMessageConverter.toMessage(data, RqueueMessageHeaders.emptyMessageHeaders()); + MultiGenericTestDataSameType fromMessage = + (MultiGenericTestDataSameType) genericMessageConverter.fromMessage(message, null); + assertEquals(data, fromMessage); + } + + @Test + @Disabled + public void testMultiLevelGenericTestDataFixedTypeToFromMessage() { + GenericTestData genericTestData = new GenericTestData<>(100, comment); + MultiGenericTestData multiGenericTestData = + new MultiGenericTestData<>(200, email, "comment"); + MultiLevelGenericTestDataFixedType data = + new MultiLevelGenericTestDataFixedType<>("10", genericTestData, multiGenericTestData); + Message message = genericMessageConverter.toMessage(data, RqueueMessageHeaders.emptyMessageHeaders()); - MultiLevelGenericTestDataNoArgs fromMessage = - (MultiLevelGenericTestDataNoArgs) + MultiLevelGenericTestDataFixedType fromMessage = + (MultiLevelGenericTestDataFixedType) genericMessageConverter.fromMessage(message, null); assertEquals(data, fromMessage); } @@ -170,13 +236,22 @@ public static class MultiLevelGenericTestData { private GenericTestData vGenericTestData; } + @Data + @AllArgsConstructor + @NoArgsConstructor + public static class MultiLevelGenericTestDataFixedType { + private String data; + private GenericTestData tGenericTestData; + private MultiGenericTestData vGenericTestData; + } + @Data @AllArgsConstructor @NoArgsConstructor public static class MultiGenericTestData { + private Integer index; private K key; private V value; - private TestData testData; } @Data @@ -190,8 +265,104 @@ public static class GenericTestData { @Data @AllArgsConstructor @NoArgsConstructor - public static class TestData { + public static class Comment { private String id; private String message; } + + @Data + @AllArgsConstructor + @NoArgsConstructor + public static class Email { + private String id; + private String subject; + } + + // https://stackoverflow.com/questions/64873444/generic-class-type-parameter-detail-at-runtime + static class MappingRegistrar { + + private final Type type; + + protected MappingRegistrar() { + Class cls = getClass(); + Type[] type = ((ParameterizedType) cls.getGenericSuperclass()).getActualTypeArguments(); + this.type = type[0]; + } + + public void seeIt() { + innerSeeIt(type); + } + + private void innerSeeIt(Type type) { + if (type instanceof Class) { + Class cls = (Class) type; + boolean isArray = cls.isArray(); + if (isArray) { + System.out.print(cls.getComponentType().getSimpleName() + "[]"); + return; + } + System.out.print(cls.getSimpleName()); + } + + if (type instanceof TypeVariable) { + Type[] bounds = ((TypeVariable) type).getBounds(); + String s = + Arrays.stream(bounds) + .map(Type::getTypeName) + .collect(Collectors.joining(", ", "[", "]")); + System.out.print(s); + } + + if (type instanceof ParameterizedType) { + ParameterizedType parameterizedType = (ParameterizedType) type; + String rawType = parameterizedType.getRawType().getTypeName(); + System.out.print(rawType + "<"); + Type[] arguments = parameterizedType.getActualTypeArguments(); + + for (int i = 0; i < arguments.length; ++i) { + innerSeeIt(arguments[i]); + if (i != arguments.length - 1) { + System.out.print(", "); + } + } + + System.out.print(">"); + // System.out.println(Arrays.toString(arguments)); + } + + if (type instanceof GenericArrayType) { + // you need to handle this one too + } + + if (type instanceof WildcardType) { + // you need to handle this one too, but it isn't trivial + } + } + } + + @Data + @AllArgsConstructor + @NoArgsConstructor + public static class MultiGenericTestDataSameType extends MappingRegistrar { + private Integer index; + private GenericTestData genericTestData; + private MultiGenericTestData multiGenericTestData; + } + + @Data + @NoArgsConstructor + @AllArgsConstructor + public static class GenericTestDataWithPredefinedType { + private Integer index; + private MultiGenericTestData data; + } + + @Test + @Disabled + public void foo() { + MappingRegistrar m = new MappingRegistrar>() {}; + m.seeIt(); + MultiGenericTestDataSameType m2 = new MultiGenericTestDataSameType<>(); + m2.seeIt(); + } } From 9b414ba0f2cbd3f8bd8eb4cdc6dc6020007772dc Mon Sep 17 00:00:00 2001 From: Sonu Kumar Date: Thu, 3 Dec 2020 00:16:12 +0530 Subject: [PATCH 07/23] Periodic job enqueing. --- .../sonus21/rqueue/core/RqueueMessage.java | 34 ++++++++++------ .../rqueue/core/RqueueMessageEnqueuer.java | 24 ++++++++++++ .../rqueue/core/impl/BaseMessageSender.java | 26 ++++++++++++- .../core/impl/RqueueMessageEnqueuerImpl.java | 18 +++++++++ .../core/impl/RqueueMessageTemplateImpl.java | 3 +- .../core/support/RqueueMessageUtils.java | 35 ++++++++++++++++- .../sonus21/rqueue/utils/BaseLogger.java | 39 +++++++++---------- .../sonus21/rqueue/utils/Validator.java | 6 +++ .../core/RqueueMessageTemplateTest.java | 4 +- .../rqueue/core/RqueueMessageTest.java | 8 +++- .../RqueueMessageListenerContainerTest.java | 18 +++++++-- .../RqueueTaskAggregatorServiceTest.java | 4 +- 12 files changed, 174 insertions(+), 45 deletions(-) diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessage.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessage.java index 53ad8f2b..1a720735 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessage.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessage.java @@ -16,6 +16,8 @@ package com.github.sonus21.rqueue.core; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; import com.github.sonus21.rqueue.models.SerializableBase; import java.util.UUID; import lombok.Getter; @@ -27,10 +29,11 @@ @Setter @ToString @NoArgsConstructor +@JsonPropertyOrder({"failureCount"}) public class RqueueMessage extends SerializableBase implements Cloneable { private static final long serialVersionUID = -3488860960637488519L; - /** The message id, each message has a unique id, generated using */ + // The message id, each message has a unique id private String id; // Queue name on which message was enqueued private String queueName; @@ -47,23 +50,25 @@ public class RqueueMessage extends SerializableBase implements Cloneable { private Long reEnqueuedAt; // Number of times this message has failed. private int failureCount; + // period of this task, if this is a periodic task. + private long period; - public RqueueMessage(String queueName, String message, Integer retryCount, Long delay) { + public RqueueMessage( + String queueName, String message, Integer retryCount, long queuedTime, long processAt) { + this.id = UUID.randomUUID().toString(); this.queueName = queueName; this.message = message; this.retryCount = retryCount; - this.id = UUID.randomUUID().toString(); - initTime(delay); + this.queuedTime = queuedTime; + this.processAt = processAt; } - private void initTime(Long delay) { - // Monotonic increasing queued time - // This is used to check duplicate message in executor - this.queuedTime = System.nanoTime(); - this.processAt = System.currentTimeMillis(); - if (delay != null) { - this.processAt += delay; - } + public RqueueMessage(String queueName, String message, long processAt, long period) { + this.id = UUID.randomUUID().toString(); + this.queueName = queueName; + this.message = message; + this.processAt = processAt; + this.period = period; } public void updateReEnqueuedAt() { @@ -86,4 +91,9 @@ public boolean equals(Object other) { } return false; } + + @JsonIgnore + public boolean isPeriodicTask() { + return period > 0; + } } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageEnqueuer.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageEnqueuer.java index 4ab3572c..d588177a 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageEnqueuer.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageEnqueuer.java @@ -600,4 +600,28 @@ default boolean enqueueUniqueAtWithPriority( return enqueueAtWithPriority( queueName, priority, messageId, message, timeInMilliSeconds - System.currentTimeMillis()); } + + /** + * Enqueue a message on given queue that will be running after a given period. It works like + * periodic cron that's scheduled at certain interval, for example every 30 seconds. + * + * @param queueName on which queue message has to be send + * @param message message object it could be any arbitrary object. + * @param periodInMilliSeconds period of this job in milliseconds. + * @return message id on successful enqueue otherwise null. + */ + String enqueuePeriodic(String queueName, Object message, long periodInMilliSeconds); + + /** + * Enqueue a message on given queue that will be running after a given period. It works like + * periodic cron that's scheduled at certain interval, for example every 30 seconds. + * + * @param queueName on which queue message has to be send + * @param messageId message id corresponding to this message + * @param message message object it could be any arbitrary object. + * @param periodInMilliSeconds period of this job in milliseconds. + * @return message id on successful enqueue otherwise null. + */ + boolean enqueuePeriodic( + String queueName, String messageId, Object message, long periodInMilliSeconds); } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/BaseMessageSender.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/BaseMessageSender.java index 808ecc99..15bc88d4 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/BaseMessageSender.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/BaseMessageSender.java @@ -17,7 +17,10 @@ package com.github.sonus21.rqueue.core.impl; import static com.github.sonus21.rqueue.core.support.RqueueMessageUtils.buildMessage; +import static com.github.sonus21.rqueue.core.support.RqueueMessageUtils.buildPeriodicMessage; import static com.github.sonus21.rqueue.utils.Constants.MIN_DELAY; +import static com.github.sonus21.rqueue.utils.Validator.validateMessage; +import static com.github.sonus21.rqueue.utils.Validator.validatePeriod; import static com.github.sonus21.rqueue.utils.Validator.validateQueue; import static org.springframework.util.Assert.notNull; @@ -58,7 +61,7 @@ abstract class BaseMessageSender { this.messageHeaders = messageHeaders; } - private void storeMessageMetadata(RqueueMessage rqueueMessage, Long delayInMillis) { + protected void storeMessageMetadata(RqueueMessage rqueueMessage, Long delayInMillis) { MessageMetadata messageMetadata = new MessageMetadata(rqueueMessage, MessageStatus.ENQUEUED); Duration duration; if (delayInMillis != null) { @@ -88,7 +91,7 @@ private RqueueMessage constructMessage( return rqueueMessage; } - private void enqueue( + protected void enqueue( QueueDetail queueDetail, RqueueMessage rqueueMessage, Long delayInMilliSecs) { if (delayInMilliSecs == null || delayInMilliSecs <= MIN_DELAY) { messageTemplate.addMessage(queueDetail.getQueueName(), rqueueMessage); @@ -119,6 +122,25 @@ protected String pushMessage( return rqueueMessage.getId(); } + protected String pushPeriodicMessage( + String queueName, String messageId, Object message, long periodInMilliSeconds) { + QueueDetail queueDetail = EndpointRegistry.get(queueName); + RqueueMessage rqueueMessage = + buildPeriodicMessage( + messageConverter, queueName, message, periodInMilliSeconds, messageHeaders); + if (messageId != null) { + rqueueMessage.setId(messageId); + } + try { + enqueue(queueDetail, rqueueMessage, periodInMilliSeconds); + storeMessageMetadata(rqueueMessage, periodInMilliSeconds); + } catch (Exception e) { + log.error("Queue: {} Message {} could not be pushed {}", queueName, rqueueMessage, e); + return null; + } + return rqueueMessage.getId(); + } + protected void registerQueueInternal(String queueName, String... priorities) { validateQueue(queueName); notNull(priorities, "priorities cannot be null"); diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageEnqueuerImpl.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageEnqueuerImpl.java index 0837e19a..26eccaff 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageEnqueuerImpl.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageEnqueuerImpl.java @@ -19,6 +19,7 @@ import static com.github.sonus21.rqueue.utils.Validator.validateDelay; import static com.github.sonus21.rqueue.utils.Validator.validateMessage; import static com.github.sonus21.rqueue.utils.Validator.validateMessageId; +import static com.github.sonus21.rqueue.utils.Validator.validatePeriod; import static com.github.sonus21.rqueue.utils.Validator.validatePriority; import static com.github.sonus21.rqueue.utils.Validator.validateQueue; import static com.github.sonus21.rqueue.utils.Validator.validateRetryCount; @@ -167,4 +168,21 @@ public boolean enqueueInWithPriority( delayInMilliSecs) != null; } + + @Override + public String enqueuePeriodic(String queueName, Object message, long period) { + validateQueue(queueName); + validateMessage(message); + validatePeriod(period); + return pushPeriodicMessage(queueName, null, message, period); + } + + @Override + public boolean enqueuePeriodic(String queueName, String messageId, Object message, long period) { + validateMessageId(messageId); + validateQueue(queueName); + validateMessage(message); + validatePeriod(period); + return pushPeriodicMessage(queueName, messageId, message, period) != null; + } } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageTemplateImpl.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageTemplateImpl.java index 6045bbbd..700216fd 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageTemplateImpl.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageTemplateImpl.java @@ -71,14 +71,13 @@ public RqueueMessage pop( @Override public Long addMessageWithDelay( String delayQueueName, String delayQueueChannelName, RqueueMessage rqueueMessage) { - long queuedTime = rqueueMessage.getQueuedTime(); RedisScript script = (RedisScript) getScript(ScriptType.ENQUEUE_MESSAGE); return scriptExecutor.execute( script, Arrays.asList(delayQueueName, delayQueueChannelName), rqueueMessage, rqueueMessage.getProcessAt(), - queuedTime); + System.currentTimeMillis()); } @Override diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/support/RqueueMessageUtils.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/support/RqueueMessageUtils.java index a17bd124..cf99f8b6 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/support/RqueueMessageUtils.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/support/RqueueMessageUtils.java @@ -45,6 +45,31 @@ public static Object convertMessageToObject( return messageConverter.fromMessage(message, null); } + public static RqueueMessage buildPeriodicMessage( + MessageConverter converter, + String queueName, + Object message, + long period, + MessageHeaders messageHeaders) { + Message msg = converter.toMessage(message, messageHeaders); + if (msg == null) { + throw new MessageConversionException("Message could not be build (null)"); + } + Object payload = msg.getPayload(); + if (payload instanceof String) { + return new RqueueMessage( + queueName, (String) payload, System.currentTimeMillis() + period, period); + } + if (payload instanceof byte[]) { + return new RqueueMessage( + queueName, + new String((byte[]) msg.getPayload()), + System.currentTimeMillis() + period, + period); + } + throw new MessageConversionException("Message payload is neither String nor byte[]"); + } + public static RqueueMessage buildMessage( MessageConverter converter, Object object, @@ -56,12 +81,18 @@ public static RqueueMessage buildMessage( if (msg == null) { throw new MessageConversionException("Message could not be build (null)"); } + long queuedTime = System.nanoTime(); + long processAt = System.currentTimeMillis(); + if (delay != null) { + processAt += delay; + } Object payload = msg.getPayload(); if (payload instanceof String) { - return new RqueueMessage(queueName, (String) payload, retryCount, delay); + return new RqueueMessage(queueName, (String) payload, retryCount, queuedTime, processAt); } if (payload instanceof byte[]) { - return new RqueueMessage(queueName, new String((byte[]) msg.getPayload()), retryCount, delay); + return new RqueueMessage( + queueName, new String((byte[]) msg.getPayload()), retryCount, queuedTime, processAt); } throw new MessageConversionException("Message payload is neither String nor byte[]"); } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/BaseLogger.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/BaseLogger.java index 084debba..8d58688b 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/BaseLogger.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/BaseLogger.java @@ -21,14 +21,18 @@ public class BaseLogger { private final Logger log; - private final String groupName; + private final String prefix; public BaseLogger(Logger log, String groupName) { this.log = log; - this.groupName = groupName; + if (StringUtils.isEmpty(groupName)) { + this.prefix = ""; + } else { + this.prefix = "[" + groupName + "] "; + } } - public void log(Level level, String msg, Throwable t, Object... objects) { + public void log(Level level, String msg, Throwable t, Object... arguments) { if (level == Level.DEBUG && !log.isDebugEnabled()) { return; } @@ -41,37 +45,32 @@ public void log(Level level, String msg, Throwable t, Object... objects) { if (level == Level.WARN && !log.isWarnEnabled()) { return; } - if (StringUtils.isEmpty(groupName)) { - logMessage(level, msg, objects); + if (t == null) { + logMessage(level, prefix + msg, arguments); } else { - int size = objects.length + 1 + (t == null ? 0 : 1); - Object[] objects1 = new Object[size]; - System.arraycopy(objects, 0, objects1, 1, objects.length); - objects1[0] = groupName; - if (t != null) { - objects1[objects1.length - 1] = t; - } - String txt = "[{}] " + msg; - logMessage(level, txt, objects1); + Object[] objects1 = new Object[arguments.length + 1]; + System.arraycopy(arguments, 0, objects1, 0, arguments.length); + objects1[objects1.length - 1] = t; + logMessage(level, prefix + msg, objects1); } } - private void logMessage(Level level, String txt, Object[] objects) { + private void logMessage(Level level, String txt, Object... arguments) { switch (level) { case INFO: - log.info(txt, objects); + log.info(txt, arguments); break; case WARN: - log.warn(txt, objects); + log.warn(txt, arguments); break; case DEBUG: - log.debug(txt, objects); + log.debug(txt, arguments); break; case ERROR: - log.error(txt, objects); + log.error(txt, arguments); break; default: - log.trace(txt, objects); + log.trace(txt, arguments); } } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/Validator.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/Validator.java index 2aa168cc..adad5c28 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/Validator.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/Validator.java @@ -55,4 +55,10 @@ public static void validatePriority(String priority) { throw new IllegalArgumentException("priority cannot be empty"); } } + + public static void validatePeriod(long period) { + if (period < Constants.ONE_MILLI) { + throw new IllegalArgumentException("period must be greater than or equal to one second."); + } + } } diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/RqueueMessageTemplateTest.java b/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/RqueueMessageTemplateTest.java index 073eab76..8de613d2 100644 --- a/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/RqueueMessageTemplateTest.java +++ b/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/RqueueMessageTemplateTest.java @@ -48,7 +48,9 @@ public class RqueueMessageTemplateTest { new RqueueMessageTemplateImpl(redisConnectionFactory); private String key = "test-queue"; - private RqueueMessage message = new RqueueMessage(key, "This is a message", null, 100L); + private RqueueMessage message = + new RqueueMessage( + key, "This is a message", null, System.nanoTime(), System.currentTimeMillis()); @BeforeEach public void init() throws Exception { diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/RqueueMessageTest.java b/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/RqueueMessageTest.java index af1e5f20..61180c89 100644 --- a/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/RqueueMessageTest.java +++ b/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/RqueueMessageTest.java @@ -37,7 +37,9 @@ public class RqueueMessageTest { @Test public void checkIdIsSetAndProcessAtIsSameAsQueuedTime() { - RqueueMessage message = new RqueueMessage(queueName, queueMessage, retryCount, null); + RqueueMessage message = + new RqueueMessage( + queueName, queueMessage, retryCount, System.nanoTime(), System.currentTimeMillis()); assertNotNull(message.getId()); assertEquals(message.getProcessAt(), message.getProcessAt()); } @@ -68,7 +70,9 @@ public void testObjectEquality() throws JsonProcessingException { @Test public void testObjectEqualityWithoutDelay() throws JsonProcessingException { - RqueueMessage message = new RqueueMessage(queueName, queueMessage, retryCount, null); + RqueueMessage message = + new RqueueMessage( + queueName, queueMessage, retryCount, System.nanoTime(), System.currentTimeMillis()); String stringMessage = objectMapper.writeValueAsString(message); assertEquals(message, objectMapper.readValue(stringMessage, RqueueMessage.class)); } diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/listener/RqueueMessageListenerContainerTest.java b/rqueue-core/src/test/java/com/github/sonus21/rqueue/listener/RqueueMessageListenerContainerTest.java index c82c2f13..2ea0f320 100644 --- a/rqueue-core/src/test/java/com/github/sonus21/rqueue/listener/RqueueMessageListenerContainerTest.java +++ b/rqueue-core/src/test/java/com/github/sonus21/rqueue/listener/RqueueMessageListenerContainerTest.java @@ -239,7 +239,9 @@ private RqueueMessageListenerContainer createContainer( public void testMessageFetcherRetryWorking() throws Exception { AtomicInteger fastQueueCounter = new AtomicInteger(0); String fastQueueMessage = "This is fast queue"; - RqueueMessage message = new RqueueMessage(fastQueue, fastQueueMessage, null, null); + RqueueMessage message = + new RqueueMessage( + fastQueue, fastQueueMessage, null, System.nanoTime(), System.currentTimeMillis()); RqueueMessageTemplate rqueueMessageTemplate = mock(RqueueMessageTemplate.class); @@ -332,7 +334,12 @@ public void testMessageHandlersAreInvoked() throws Exception { invocation -> { if (slowQueueCounter.get() == 0) { slowQueueCounter.incrementAndGet(); - return new RqueueMessage(slowQueue, slowQueueMessage, null, null); + return new RqueueMessage( + slowQueue, + slowQueueMessage, + null, + System.nanoTime(), + System.currentTimeMillis()); } return null; }) @@ -343,7 +350,12 @@ public void testMessageHandlersAreInvoked() throws Exception { invocation -> { if (fastQueueCounter.get() == 0) { fastQueueCounter.incrementAndGet(); - return new RqueueMessage(fastQueue, fastQueueMessage, null, null); + return new RqueueMessage( + fastQueue, + fastQueueMessage, + null, + System.nanoTime(), + System.currentTimeMillis()); } return null; }) diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/web/service/RqueueTaskAggregatorServiceTest.java b/rqueue-core/src/test/java/com/github/sonus21/rqueue/web/service/RqueueTaskAggregatorServiceTest.java index 3a7b31ba..93cc3f7e 100644 --- a/rqueue-core/src/test/java/com/github/sonus21/rqueue/web/service/RqueueTaskAggregatorServiceTest.java +++ b/rqueue-core/src/test/java/com/github/sonus21/rqueue/web/service/RqueueTaskAggregatorServiceTest.java @@ -79,7 +79,9 @@ public void initService() throws IllegalAccessException { private RqueueExecutionEvent generateTaskEventWithStatus(TaskStatus status) { double r = Math.random(); - RqueueMessage rqueueMessage = new RqueueMessage("test-queue", "test", null, null); + RqueueMessage rqueueMessage = + new RqueueMessage( + "test-queue", "test", null, System.nanoTime(), System.currentTimeMillis()); MessageMetadata messageMetadata = new MessageMetadata(rqueueMessage.getId(), MessageStatus.FAILED); messageMetadata.setTotalExecutionTime(10 + (long) r * 10000); From 991d80d38a5d50f3f32a6512da18d4963d4b17a2 Mon Sep 17 00:00:00 2001 From: Sonu Kumar Date: Thu, 3 Dec 2020 08:53:22 +0530 Subject: [PATCH 08/23] Removed RqueueMessage constructors. --- .../sonus21/rqueue/core/RqueueMessage.java | 23 +--- .../core/support/RqueueMessageUtils.java | 41 +++++-- .../GenericMessageConverterTest.java | 2 + .../core/RqueueMessageTemplateTest.java | 52 ++++---- .../rqueue/core/RqueueMessageTest.java | 114 ++++++++++++------ .../RqueueMessageListenerContainerTest.java | 35 +++--- .../RqueueTaskAggregatorServiceTest.java | 8 +- 7 files changed, 172 insertions(+), 103 deletions(-) diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessage.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessage.java index 1a720735..0ea6c5b2 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessage.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessage.java @@ -19,7 +19,8 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonPropertyOrder; import com.github.sonus21.rqueue.models.SerializableBase; -import java.util.UUID; +import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; @@ -29,7 +30,9 @@ @Setter @ToString @NoArgsConstructor +@AllArgsConstructor @JsonPropertyOrder({"failureCount"}) +@Builder public class RqueueMessage extends SerializableBase implements Cloneable { private static final long serialVersionUID = -3488860960637488519L; @@ -53,24 +56,6 @@ public class RqueueMessage extends SerializableBase implements Cloneable { // period of this task, if this is a periodic task. private long period; - public RqueueMessage( - String queueName, String message, Integer retryCount, long queuedTime, long processAt) { - this.id = UUID.randomUUID().toString(); - this.queueName = queueName; - this.message = message; - this.retryCount = retryCount; - this.queuedTime = queuedTime; - this.processAt = processAt; - } - - public RqueueMessage(String queueName, String message, long processAt, long period) { - this.id = UUID.randomUUID().toString(); - this.queueName = queueName; - this.message = message; - this.processAt = processAt; - this.period = period; - } - public void updateReEnqueuedAt() { reEnqueuedAt = System.currentTimeMillis(); } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/support/RqueueMessageUtils.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/support/RqueueMessageUtils.java index cf99f8b6..a79a79a1 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/support/RqueueMessageUtils.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/support/RqueueMessageUtils.java @@ -19,6 +19,7 @@ import com.github.sonus21.rqueue.core.RqueueMessage; import java.util.ArrayList; import java.util.List; +import java.util.UUID; import org.springframework.messaging.Message; import org.springframework.messaging.MessageHeaders; import org.springframework.messaging.converter.MessageConversionException; @@ -57,15 +58,22 @@ public static RqueueMessage buildPeriodicMessage( } Object payload = msg.getPayload(); if (payload instanceof String) { - return new RqueueMessage( - queueName, (String) payload, System.currentTimeMillis() + period, period); + return RqueueMessage.builder() + .period(period) + .id(UUID.randomUUID().toString()) + .queueName(queueName) + .message((String) payload) + .processAt(System.currentTimeMillis() + period) + .build(); } if (payload instanceof byte[]) { - return new RqueueMessage( - queueName, - new String((byte[]) msg.getPayload()), - System.currentTimeMillis() + period, - period); + return RqueueMessage.builder() + .period(period) + .id(UUID.randomUUID().toString()) + .queueName(queueName) + .message(new String((byte[]) msg.getPayload())) + .processAt(System.currentTimeMillis() + period) + .build(); } throw new MessageConversionException("Message payload is neither String nor byte[]"); } @@ -88,11 +96,24 @@ public static RqueueMessage buildMessage( } Object payload = msg.getPayload(); if (payload instanceof String) { - return new RqueueMessage(queueName, (String) payload, retryCount, queuedTime, processAt); + return RqueueMessage.builder() + .retryCount(retryCount) + .queuedTime(queuedTime) + .id(UUID.randomUUID().toString()) + .queueName(queueName) + .message((String) payload) + .processAt(processAt) + .build(); } if (payload instanceof byte[]) { - return new RqueueMessage( - queueName, new String((byte[]) msg.getPayload()), retryCount, queuedTime, processAt); + return RqueueMessage.builder() + .retryCount(retryCount) + .queuedTime(queuedTime) + .id(UUID.randomUUID().toString()) + .queueName(queueName) + .message(new String((byte[]) msg.getPayload())) + .processAt(processAt) + .build(); } throw new MessageConversionException("Message payload is neither String nor byte[]"); } diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/converter/GenericMessageConverterTest.java b/rqueue-core/src/test/java/com/github/sonus21/rqueue/converter/GenericMessageConverterTest.java index 8976f660..3c27d03e 100644 --- a/rqueue-core/src/test/java/com/github/sonus21/rqueue/converter/GenericMessageConverterTest.java +++ b/rqueue-core/src/test/java/com/github/sonus21/rqueue/converter/GenericMessageConverterTest.java @@ -33,6 +33,7 @@ import java.util.stream.Collectors; import lombok.AllArgsConstructor; import lombok.Data; +import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @@ -340,6 +341,7 @@ private void innerSeeIt(Type type) { } } + @EqualsAndHashCode(callSuper = true) @Data @AllArgsConstructor @NoArgsConstructor diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/RqueueMessageTemplateTest.java b/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/RqueueMessageTemplateTest.java index 8de613d2..a337115d 100644 --- a/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/RqueueMessageTemplateTest.java +++ b/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/RqueueMessageTemplateTest.java @@ -27,6 +27,7 @@ import com.github.sonus21.rqueue.utils.Constants; import java.util.ArrayList; import java.util.List; +import java.util.UUID; import org.apache.commons.lang3.reflect.FieldUtils; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -38,19 +39,23 @@ import org.springframework.data.redis.core.script.DefaultScriptExecutor; @ExtendWith(MockitoExtension.class) +@SuppressWarnings("unchecked") public class RqueueMessageTemplateTest { - private RedisConnectionFactory redisConnectionFactory = mock(RedisConnectionFactory.class); - private RedisTemplate redisTemplate = mock(RedisTemplate.class); - private ListOperations listOperations = mock(ListOperations.class); - private DefaultScriptExecutor scriptExecutor = mock(DefaultScriptExecutor.class); - - private RqueueMessageTemplate rqueueMessageTemplate = + private final RedisConnectionFactory redisConnectionFactory = mock(RedisConnectionFactory.class); + private final RedisTemplate redisTemplate = mock(RedisTemplate.class); + private final ListOperations listOperations = mock(ListOperations.class); + private final DefaultScriptExecutor scriptExecutor = mock(DefaultScriptExecutor.class); + private final RqueueMessageTemplate rqueueMessageTemplate = new RqueueMessageTemplateImpl(redisConnectionFactory); - - private String key = "test-queue"; - private RqueueMessage message = - new RqueueMessage( - key, "This is a message", null, System.nanoTime(), System.currentTimeMillis()); + private final String queueName = "test-queue"; + private final RqueueMessage message = + RqueueMessage.builder() + .queuedTime(System.nanoTime()) + .id(UUID.randomUUID().toString()) + .queueName(queueName) + .message("This is a test message") + .processAt(System.currentTimeMillis()) + .build(); @BeforeEach public void init() throws Exception { @@ -61,27 +66,28 @@ public void init() throws Exception { @Test public void add() { doReturn(listOperations).when(redisTemplate).opsForList(); - doReturn(1L).when(listOperations).rightPush(key, message); - rqueueMessageTemplate.addMessage(key, message); + doReturn(1L).when(listOperations).rightPush(queueName, message); + rqueueMessageTemplate.addMessage(queueName, message); } @Test public void pop() { - rqueueMessageTemplate.pop(key, key + "rq", key + "dq", Constants.DELTA_BETWEEN_RE_ENQUEUE_TIME); + rqueueMessageTemplate.pop( + queueName, queueName + "rq", queueName + "dq", Constants.DELTA_BETWEEN_RE_ENQUEUE_TIME); verify(scriptExecutor, times(1)).execute(any(), any(), any()); } @Test public void addWithDelay() { - rqueueMessageTemplate.addMessageWithDelay(key, key + "rq", message); + rqueueMessageTemplate.addMessageWithDelay(queueName, queueName + "rq", message); verify(scriptExecutor, times(1)).execute(any(), any(), any()); } @Test public void moveMessage() { List args = new ArrayList<>(); - args.add("dlq" + key); - args.add(key); + args.add("dlq" + queueName); + args.add(queueName); doReturn(70L).when(scriptExecutor).execute(any(), eq(args), eq(100L)); doReturn(20L).when(scriptExecutor).execute(any(), eq(args), eq(50L)); rqueueMessageTemplate.moveMessageListToList(args.get(0), args.get(1), 150); @@ -92,8 +98,8 @@ public void moveMessage() { @Test public void moveMessage2() { List args = new ArrayList<>(); - args.add("dlq" + key); - args.add(key); + args.add("dlq" + queueName); + args.add(queueName); doReturn(0L).when(scriptExecutor).execute(any(), eq(args), eq(100L)); rqueueMessageTemplate.moveMessageListToList(args.get(0), args.get(1), 150); verify(scriptExecutor, times(1)).execute(any(), eq(args), eq(100L)); @@ -102,8 +108,8 @@ public void moveMessage2() { @Test public void moveMessageAcrossZset() { List args = new ArrayList<>(); - args.add("zset1-" + key); - args.add("zset2-" + key); + args.add("zset1-" + queueName); + args.add("zset2-" + queueName); doReturn(0L).when(scriptExecutor).execute(any(), eq(args), eq(100L), eq(10L), eq(false)); rqueueMessageTemplate.moveMessageZsetToZset(args.get(0), args.get(1), 150, 10L, false); } @@ -111,8 +117,8 @@ public void moveMessageAcrossZset() { @Test public void moveMessageAcrossZset2() { List args = new ArrayList<>(); - args.add("zset1-" + key); - args.add("zset2-" + key); + args.add("zset1-" + queueName); + args.add("zset2-" + queueName); long score = System.currentTimeMillis(); doReturn(70L).when(scriptExecutor).execute(any(), eq(args), eq(100L), eq(score), eq(true)); doReturn(20L).when(scriptExecutor).execute(any(), eq(args), eq(50L), eq(score), eq(true)); diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/RqueueMessageTest.java b/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/RqueueMessageTest.java index 61180c89..042f7a6c 100644 --- a/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/RqueueMessageTest.java +++ b/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/RqueueMessageTest.java @@ -17,12 +17,13 @@ package com.github.sonus21.rqueue.core; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.UUID; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.junit.jupiter.MockitoExtension; @@ -35,27 +36,16 @@ public class RqueueMessageTest { private Integer retryCount = 3; private long delay = 100L; - @Test - public void checkIdIsSetAndProcessAtIsSameAsQueuedTime() { - RqueueMessage message = - new RqueueMessage( - queueName, queueMessage, retryCount, System.nanoTime(), System.currentTimeMillis()); - assertNotNull(message.getId()); - assertEquals(message.getProcessAt(), message.getProcessAt()); - } - - @Test - public void checkIdAndProcessAtAreSet() { - RqueueMessage message = new RqueueMessage(queueName, queueMessage, retryCount, delay); - assertNotNull(message.getId()); - assertTrue( - message.getProcessAt() <= System.currentTimeMillis() + 100 - && message.getProcessAt() > System.currentTimeMillis()); - } - @Test public void testSetReEnqueuedAt() { - RqueueMessage message = new RqueueMessage(queueName, queueMessage, retryCount, delay); + RqueueMessage message = + RqueueMessage.builder() + .queueName(queueName) + .message(queueMessage) + .retryCount(retryCount) + .processAt(System.currentTimeMillis() + delay) + .queuedTime(System.nanoTime()) + .build(); Long time = System.currentTimeMillis() - delay; message.setReEnqueuedAt(time); assertEquals(message.getReEnqueuedAt(), time); @@ -63,7 +53,15 @@ public void testSetReEnqueuedAt() { @Test public void testObjectEquality() throws JsonProcessingException { - RqueueMessage message = new RqueueMessage(queueName, queueMessage, retryCount, delay); + RqueueMessage message = + RqueueMessage.builder() + .id(UUID.randomUUID().toString()) + .queueName(queueName) + .message(queueMessage) + .retryCount(retryCount) + .processAt(System.currentTimeMillis() + delay) + .queuedTime(System.nanoTime()) + .build(); String stringMessage = objectMapper.writeValueAsString(message); assertEquals(message, objectMapper.readValue(stringMessage, RqueueMessage.class)); } @@ -71,35 +69,66 @@ public void testObjectEquality() throws JsonProcessingException { @Test public void testObjectEqualityWithoutDelay() throws JsonProcessingException { RqueueMessage message = - new RqueueMessage( - queueName, queueMessage, retryCount, System.nanoTime(), System.currentTimeMillis()); + RqueueMessage.builder() + .id(UUID.randomUUID().toString()) + .queueName(queueName) + .message(queueMessage) + .retryCount(retryCount) + .processAt(System.currentTimeMillis()) + .queuedTime(System.nanoTime()) + .build(); String stringMessage = objectMapper.writeValueAsString(message); assertEquals(message, objectMapper.readValue(stringMessage, RqueueMessage.class)); } @Test public void testObjectEqualityWithDifferentObject() { - RqueueMessage message = new RqueueMessage(queueName, queueMessage, retryCount, delay); + RqueueMessage message = + RqueueMessage.builder() + .queueName(queueName) + .message(queueMessage) + .retryCount(retryCount) + .processAt(System.currentTimeMillis()) + .queuedTime(System.nanoTime()) + .build(); assertNotEquals(message, new Object()); } @Test public void testObjectEqualityWithDifferentId() { - RqueueMessage message = new RqueueMessage(queueName, queueMessage, retryCount, delay); - RqueueMessage message2 = new RqueueMessage(queueName + "2", queueMessage, retryCount, delay); - assertNotEquals(message, message2); - } + RqueueMessage message = + RqueueMessage.builder() + .id(UUID.randomUUID().toString()) + .queueName(queueName) + .message(queueMessage) + .retryCount(retryCount) + .processAt(System.currentTimeMillis()) + .queuedTime(System.nanoTime()) + .build(); - @Test - public void testObjectEqualityWithDifferentMessageContent() { - RqueueMessage message = new RqueueMessage(queueName, queueMessage + 1, retryCount, delay); - RqueueMessage message2 = new RqueueMessage(queueName, queueMessage + 2, retryCount, delay); + RqueueMessage message2 = + RqueueMessage.builder() + .id("x" + UUID.randomUUID().toString()) + .queueName(queueName) + .message(queueMessage) + .retryCount(retryCount) + .processAt(System.currentTimeMillis()) + .queuedTime(System.nanoTime()) + .build(); assertNotEquals(message, message2); } @Test public void testToString() { - RqueueMessage message = new RqueueMessage(queueName, queueMessage, retryCount, delay); + RqueueMessage message = + RqueueMessage.builder() + .id(UUID.randomUUID().toString()) + .queueName(queueName) + .message(queueMessage) + .retryCount(retryCount) + .processAt(System.currentTimeMillis()) + .queuedTime(System.nanoTime()) + .build(); String toString = "RqueueMessage(id=" + message.getId() @@ -109,7 +138,24 @@ public void testToString() { + message.getQueuedTime() + ", processAt=" + message.getProcessAt() - + ", reEnqueuedAt=null, failureCount=0)"; + + ", reEnqueuedAt=null, failureCount=0, period=0)"; assertEquals(toString, message.toString()); } + + @Test + public void testIsPeriodic() { + RqueueMessage message = + RqueueMessage.builder() + .id(UUID.randomUUID().toString()) + .queueName(queueName) + .message(queueMessage) + .retryCount(retryCount) + .processAt(System.currentTimeMillis()) + .queuedTime(System.nanoTime()) + .period(10000) + .build(); + assertTrue(message.isPeriodicTask()); + message.setPeriod(0); + assertFalse(message.isPeriodicTask()); + } } diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/listener/RqueueMessageListenerContainerTest.java b/rqueue-core/src/test/java/com/github/sonus21/rqueue/listener/RqueueMessageListenerContainerTest.java index 2ea0f320..56326afa 100644 --- a/rqueue-core/src/test/java/com/github/sonus21/rqueue/listener/RqueueMessageListenerContainerTest.java +++ b/rqueue-core/src/test/java/com/github/sonus21/rqueue/listener/RqueueMessageListenerContainerTest.java @@ -34,6 +34,7 @@ import com.github.sonus21.rqueue.web.service.RqueueMessageMetadataService; import io.lettuce.core.RedisCommandExecutionException; import java.util.Map; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.Future; @@ -240,9 +241,13 @@ public void testMessageFetcherRetryWorking() throws Exception { AtomicInteger fastQueueCounter = new AtomicInteger(0); String fastQueueMessage = "This is fast queue"; RqueueMessage message = - new RqueueMessage( - fastQueue, fastQueueMessage, null, System.nanoTime(), System.currentTimeMillis()); - + RqueueMessage.builder() + .id(UUID.randomUUID().toString()) + .queueName(fastQueue) + .message(fastQueueMessage) + .processAt(System.currentTimeMillis()) + .queuedTime(System.nanoTime()) + .build(); RqueueMessageTemplate rqueueMessageTemplate = mock(RqueueMessageTemplate.class); StaticApplicationContext applicationContext = new StaticApplicationContext(); @@ -334,12 +339,12 @@ public void testMessageHandlersAreInvoked() throws Exception { invocation -> { if (slowQueueCounter.get() == 0) { slowQueueCounter.incrementAndGet(); - return new RqueueMessage( - slowQueue, - slowQueueMessage, - null, - System.nanoTime(), - System.currentTimeMillis()); + return RqueueMessage.builder() + .queueName(slowQueue) + .message(slowQueueMessage) + .processAt(System.currentTimeMillis()) + .queuedTime(System.nanoTime()) + .build(); } return null; }) @@ -350,12 +355,12 @@ public void testMessageHandlersAreInvoked() throws Exception { invocation -> { if (fastQueueCounter.get() == 0) { fastQueueCounter.incrementAndGet(); - return new RqueueMessage( - fastQueue, - fastQueueMessage, - null, - System.nanoTime(), - System.currentTimeMillis()); + return RqueueMessage.builder() + .queueName(fastQueue) + .message(fastQueueMessage) + .processAt(System.currentTimeMillis()) + .queuedTime(System.nanoTime()) + .build(); } return null; }) diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/web/service/RqueueTaskAggregatorServiceTest.java b/rqueue-core/src/test/java/com/github/sonus21/rqueue/web/service/RqueueTaskAggregatorServiceTest.java index 93cc3f7e..177957fb 100644 --- a/rqueue-core/src/test/java/com/github/sonus21/rqueue/web/service/RqueueTaskAggregatorServiceTest.java +++ b/rqueue-core/src/test/java/com/github/sonus21/rqueue/web/service/RqueueTaskAggregatorServiceTest.java @@ -80,8 +80,12 @@ public void initService() throws IllegalAccessException { private RqueueExecutionEvent generateTaskEventWithStatus(TaskStatus status) { double r = Math.random(); RqueueMessage rqueueMessage = - new RqueueMessage( - "test-queue", "test", null, System.nanoTime(), System.currentTimeMillis()); + RqueueMessage.builder() + .queueName("test-queue") + .message("test") + .processAt(System.currentTimeMillis()) + .queuedTime(System.nanoTime()) + .build(); MessageMetadata messageMetadata = new MessageMetadata(rqueueMessage.getId(), MessageStatus.FAILED); messageMetadata.setTotalExecutionTime(10 + (long) r * 10000); From b66d2cf4c6cce086abddd4d39e01be28c3fcf80d Mon Sep 17 00:00:00 2001 From: Sonu Kumar Date: Thu, 3 Dec 2020 09:27:49 +0530 Subject: [PATCH 09/23] log execution time warning. --- .../rqueue/listener/RqueueExecutor.java | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueExecutor.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueExecutor.java index 0aeb57e0..4f44d891 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueExecutor.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueExecutor.java @@ -104,10 +104,12 @@ private void updateCounter(boolean fail) { } } + private long maxExecutionTime() { + return queueDetail.getVisibilityTimeout() - DELTA_BETWEEN_RE_ENQUEUE_TIME; + } + private long getMaxProcessingTime() { - return System.currentTimeMillis() - + queueDetail.getVisibilityTimeout() - - DELTA_BETWEEN_RE_ENQUEUE_TIME; + return System.currentTimeMillis() + maxExecutionTime(); } private boolean isMessageDeleted() { @@ -167,6 +169,22 @@ private void updateToProcessing() { messageMetadata, Duration.ofMinutes(rqueueConfig.getMessageDurabilityInMinute())); } + private void logExecutionTimeWarning( + long maxProcessingTime, long startTime, ExecutionStatus status) { + if (System.currentTimeMillis() > maxProcessingTime) { + long maxAllowedTime = maxExecutionTime(); + long executionTime = System.currentTimeMillis() - startTime; + log( + Level.WARN, + "Message listener is taking longer time [Queue: {}, TaskStatus: {}] MaxAllowedTime: {}, ExecutionTime: {}", + null, + queueDetail.getQueueName(), + status, + maxAllowedTime, + executionTime); + } + } + @Override void start() { int failureCount = rqueueMessage.getFailureCount(); @@ -206,6 +224,7 @@ void start() { (status == null ? ExecutionStatus.FAILED : status), failureCount, startTime); + logExecutionTimeWarning(maxProcessingTime, startTime, status); } finally { semaphore.release(); } From 09221a0965dad6ede512ad5df7efda6fb45eaa78 Mon Sep 17 00:00:00 2001 From: Sonu Kumar Date: Thu, 3 Dec 2020 19:35:51 +0530 Subject: [PATCH 10/23] periodic job #51 --- README.md | 9 +- .../sonus21/rqueue/test/MessageListener.java | 69 +++++++++++++--- .../rqueue/test/common/SpringTestBase.java | 11 +++ .../sonus21/rqueue/test/dto/PeriodicJob.java | 37 +++++++++ .../rqueue/test/entity/ConsumedMessage.java | 28 ++++++- .../repository/ConsumedMessageRepository.java | 12 ++- .../test/service/ConsumedMessageService.java | 48 +++++++---- .../src/main/resources/application.properties | 4 + .../src/main/resources/logback.xml | 2 +- .../rqueue/core/RedisScriptFactory.java | 4 +- .../sonus21/rqueue/core/RqueueMessage.java | 23 ++++-- .../rqueue/core/RqueueMessageEnqueuer.java | 60 +++++++++++++- .../rqueue/core/RqueueMessageTemplate.java | 2 + .../core/impl/RqueueMessageTemplateImpl.java | 12 +++ .../listener/PostProcessingHandler.java | 68 ++++++--------- .../rqueue/listener/RqueueExecutor.java | 21 ++++- .../sonus21/rqueue/utils/ValueResolver.java | 4 +- .../resources/scripts/schedule_message.lua | 11 +++ .../integration/MessageSchedulingTest.java | 21 +++++ .../integration/PeriodicMessageTest.java | 82 +++++++++++++++++++ .../ConsumedMessageRepositoryImpl.java | 22 +++++ 21 files changed, 465 insertions(+), 85 deletions(-) create mode 100644 rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/dto/PeriodicJob.java create mode 100644 rqueue-core/src/main/resources/scripts/schedule_message.lua create mode 100644 rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/MessageSchedulingTest.java create mode 100644 rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/PeriodicMessageTest.java diff --git a/README.md b/README.md index cd749ef3..f2644dd7 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ * **Message delivery**: It's guaranteed that a message is consumed **at least once**. (Message would be consumed by a worker more than once due to the failure in the underlying worker/restart-process etc, otherwise exactly one delivery) * **Redis cluster** : Redis cluster can be used with driver. * **Metrics** : In flight messages, waiting for consumption and delayed messages -* **Web interface**: a web interface to manage a queue and queue insights including latency +* **Web Dashboard**: a web dashboard to manage a queue and queue insights including latency * **Automatic message serialization and deserialization** * **Concurrency**: Concurrency of any queue can be configured * **Queue Priority** : @@ -28,6 +28,7 @@ * **Callbacks** : Callbacks for dead letter queue, discard etc * **Events** 1. Bootstrap event 2. Task execution event. * **Unique message** : Unique message processing for a queue based on the message id +* **Periodic message** : Process same message at certain interval * **Redis connection**: A different redis setup can be used for Rqueue ## Getting Started @@ -117,6 +118,12 @@ public class MessageService { public void sendSms(Sms sms, SmsPriority priority){ rqueueMessageEnqueuer.enqueueWithPriority("sms-queue", priority.value(), sms); } + + // enqueue periodic job, email should be sent every 30 seconds + public void sendPeriodicEmail(Email email){ + rqueueMessageEnqueuer.enqueuePeriodic("email-queue", invoice, 30_000); + } + } ``` diff --git a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/MessageListener.java b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/MessageListener.java index d5548cef..f5185325 100644 --- a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/MessageListener.java +++ b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/MessageListener.java @@ -25,6 +25,7 @@ import com.github.sonus21.rqueue.test.dto.FeedGeneration; import com.github.sonus21.rqueue.test.dto.Job; import com.github.sonus21.rqueue.test.dto.Notification; +import com.github.sonus21.rqueue.test.dto.PeriodicJob; import com.github.sonus21.rqueue.test.dto.Reservation; import com.github.sonus21.rqueue.test.dto.ReservationRequest; import com.github.sonus21.rqueue.test.dto.Sms; @@ -36,6 +37,7 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.messaging.handler.annotation.Header; import org.springframework.messaging.handler.annotation.Payload; import org.springframework.stereotype.Component; @@ -49,13 +51,46 @@ public class MessageListener { @NonNull private ConsumedMessageService consumedMessageService; @NonNull private FailureManager failureManager; + @Value("${job.queue.name}") + private String jobQueue; + + @Value("${notification.queue.name}") + private String notificationQueueName; + + @Value("${email.queue.name}") + private String emailQueue; + + @Value("${sms.queue}") + private String smsQueue; + + @Value("${chat.indexing.queue}") + private String chatIndexingQueue; + + @Value("${feed.generation.queue}") + private String feedGenerationQueue; + + @Value("${reservation.queue}") + private String reservationQueue; + + @Value("${reservation.request.queue.name}") + private String reservationRequestQueue; + + @Value("${reservation.request.dead.letter.queue.name}") + private String reservationRequestDeadLetterQueue; + + @Value("${list.email.queue.name}") + private String listEmailQueue; + + @Value("${periodic.job.queue.name}") + private String periodicJobQueue; + @RqueueListener(value = "${job.queue.name}", active = "${job.queue.active}") public void onMessage(Job job) throws Exception { log.info("Job: {}", job); if (failureManager.shouldFail(job.getId())) { throw new Exception("Failing job task to be retried" + job); } - consumedMessageService.save(job, null); + consumedMessageService.save(job, null, jobQueue); } @RqueueListener( @@ -69,7 +104,7 @@ public void onMessage( if (failureManager.shouldFail(notification.getId())) { throw new Exception("Failing notification task to be retried" + notification); } - consumedMessageService.save(notification, null); + consumedMessageService.save(notification, null, notificationQueueName); } @RqueueListener( @@ -84,7 +119,7 @@ public void onMessage(Email email, @Header(RqueueMessageHeaders.MESSAGE) RqueueM if (failureManager.shouldFail(email.getId())) { throw new Exception("Failing email task to be retried" + email); } - consumedMessageService.save(email, null); + consumedMessageService.save(email, null, emailQueue); } @RqueueListener( @@ -98,7 +133,7 @@ public void onMessage(Sms sms) throws Exception { if (failureManager.shouldFail(sms.getId())) { throw new Exception("Failing sms task to be retried" + sms); } - consumedMessageService.save(sms, null); + consumedMessageService.save(sms, null, smsQueue); } @RqueueListener( @@ -112,7 +147,7 @@ public void onMessage(ChatIndexing chatIndexing) throws Exception { if (failureManager.shouldFail(chatIndexing.getId())) { throw new Exception("Failing chat indexing task to be retried" + chatIndexing); } - consumedMessageService.save(chatIndexing, null); + consumedMessageService.save(chatIndexing, null, chatIndexingQueue); } @RqueueListener( @@ -126,7 +161,7 @@ public void onMessage(FeedGeneration feedGeneration) throws Exception { if (failureManager.shouldFail(feedGeneration.getId())) { throw new Exception("Failing feedGeneration task to be retried" + feedGeneration); } - consumedMessageService.save(feedGeneration, null); + consumedMessageService.save(feedGeneration, null, feedGenerationQueue); } @RqueueListener( @@ -140,7 +175,7 @@ public void onMessage(Reservation reservation) throws Exception { if (failureManager.shouldFail(reservation.getId())) { throw new Exception("Failing reservation task to be retried" + reservation); } - consumedMessageService.save(reservation, null); + consumedMessageService.save(reservation, null, reservationQueue); } @RqueueListener( @@ -154,7 +189,7 @@ public void onMessageReservationRequest(ReservationRequest request) throws Excep if (failureManager.shouldFail(request.getId())) { throw new Exception("Failing reservation request task to be retried" + request); } - consumedMessageService.save(request, null); + consumedMessageService.save(request, null, reservationRequestQueue); } @RqueueListener( @@ -164,7 +199,8 @@ public void onMessageReservationRequest(ReservationRequest request) throws Excep public void onMessageReservationRequestDeadLetterQueue(ReservationRequest request) throws Exception { log.info("ReservationRequest Dead Letter Queue{}", request); - consumedMessageService.save(request, "reservation-request-dlq"); + consumedMessageService.save( + request, "reservation-request-dlq", reservationRequestDeadLetterQueue); } @RqueueListener(value = "${list.email.queue.name}", active = "${list.email.queue.enabled}") @@ -172,7 +208,20 @@ public void onMessageEmailList(List emailList) throws JsonProcessingExcep log.info("onMessageEmailList {}", emailList); String consumedId = UUID.randomUUID().toString(); for (Email email : emailList) { - consumedMessageService.save(email, consumedId); + consumedMessageService.save(email, consumedId, listEmailQueue); + } + } + + @RqueueListener( + value = "${periodic.job.queue.name}", + active = "${periodic.job.queue.active}", + deadLetterQueue = "${periodic.job.dead.letter.queue.name}", + numRetries = "${periodic.job.queue.retry.count}") + public void onPeriodicJob(PeriodicJob periodicJob) throws Exception { + log.info("onPeriodicJob: {}", periodicJob); + if (failureManager.shouldFail(periodicJob.getId())) { + throw new Exception("Failing PeriodicJob task to be retried" + periodicJob); } + consumedMessageService.save(periodicJob, UUID.randomUUID().toString(), periodicJobQueue); } } diff --git a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/common/SpringTestBase.java b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/common/SpringTestBase.java index b20cc2ee..a36ccb74 100644 --- a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/common/SpringTestBase.java +++ b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/common/SpringTestBase.java @@ -29,6 +29,7 @@ import com.github.sonus21.rqueue.core.support.RqueueMessageUtils; import com.github.sonus21.rqueue.listener.QueueDetail; import com.github.sonus21.rqueue.listener.RqueueMessageListenerContainer; +import com.github.sonus21.rqueue.test.entity.ConsumedMessage; import com.github.sonus21.rqueue.test.service.ConsumedMessageService; import com.github.sonus21.rqueue.test.service.FailureManager; import com.github.sonus21.rqueue.utils.StringUtils; @@ -101,6 +102,9 @@ public abstract class SpringTestBase extends TestBase { @Value("${list.email.queue.name}") protected String listEmailQueue; + @Value("${periodic.job.queue.name}") + protected String periodicJobQueue; + protected void enqueue(Object message, String queueName) { RqueueMessage rqueueMessage = RqueueMessageUtils.buildMessage( @@ -185,6 +189,13 @@ protected void printQueueStats(String queueName) { printQueueStats(Collections.singletonList(queueName)); } + protected void printConsumedMessage(String queueName) { + for (ConsumedMessage consumedMessage : + consumedMessageService.getConsumedMessagesForQueue(queueName)) { + log.info("Queue {} Msg: {}", queueName, consumedMessage); + } + } + protected void cleanQueue(String queue) { QueueDetail queueDetail = EndpointRegistry.get(queue); stringRqueueRedisTemplate.delete(queueDetail.getQueueName()); diff --git a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/dto/PeriodicJob.java b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/dto/PeriodicJob.java new file mode 100644 index 00000000..d57344d1 --- /dev/null +++ b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/dto/PeriodicJob.java @@ -0,0 +1,37 @@ +/* + * Copyright 2020 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.sonus21.rqueue.test.dto; + +import java.util.UUID; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.apache.commons.lang3.RandomStringUtils; + +@Data +@NoArgsConstructor +@EqualsAndHashCode(callSuper = true) +public class PeriodicJob extends BaseQueueMessage { + private String jobName; + + public static PeriodicJob newInstance() { + PeriodicJob job = new PeriodicJob(); + job.setId(UUID.randomUUID().toString()); + job.setJobName(RandomStringUtils.randomAlphabetic(10)); + return job; + } +} diff --git a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/entity/ConsumedMessage.java b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/entity/ConsumedMessage.java index 67568801..21825469 100644 --- a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/entity/ConsumedMessage.java +++ b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/entity/ConsumedMessage.java @@ -18,13 +18,17 @@ import javax.persistence.Column; import javax.persistence.Entity; +import javax.persistence.GeneratedValue; import javax.persistence.Id; +import javax.persistence.Table; +import javax.persistence.UniqueConstraint; import lombok.AllArgsConstructor; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; import lombok.ToString; +import org.hibernate.annotations.GenericGenerator; @AllArgsConstructor @NoArgsConstructor @@ -33,12 +37,32 @@ @ToString @EqualsAndHashCode @Entity +@Table( + name = "consumed_messages", + uniqueConstraints = {@UniqueConstraint(columnNames = {"message_id", "tag"})}) public class ConsumedMessage { - @Id private String id; + + @Id + @GeneratedValue(generator = "UUID") + @GenericGenerator(name = "UUID", strategy = "org.hibernate.id.UUIDGenerator") + @Column + private String id; + + @Column(name = "message_id") + private String messageId; + + @Column(name = "tag") + private String tag; + + private String queueName; // Around 1 MB of data @Column(length = 1000000) private String message; - @Column private String tag; + @Column private Long createdAt; + + public ConsumedMessage(String messageId, String tag, String queueName, String message) { + this(null, messageId, tag, queueName, message, System.currentTimeMillis()); + } } diff --git a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/repository/ConsumedMessageRepository.java b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/repository/ConsumedMessageRepository.java index 04019f66..b0126ad6 100644 --- a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/repository/ConsumedMessageRepository.java +++ b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/repository/ConsumedMessageRepository.java @@ -17,6 +17,16 @@ package com.github.sonus21.rqueue.test.repository; import com.github.sonus21.rqueue.test.entity.ConsumedMessage; +import java.util.Collection; +import java.util.List; import org.springframework.data.repository.CrudRepository; -public interface ConsumedMessageRepository extends CrudRepository {} +public interface ConsumedMessageRepository extends CrudRepository { + List findByQueueName(String queueName); + + List findByMessageId(String messageId); + + List findByMessageIdIn(Collection messageIds); + + ConsumedMessage findByMessageIdAndTag(String messageId, String tag); +} diff --git a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/service/ConsumedMessageService.java b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/service/ConsumedMessageService.java index 05a85eb3..d8718a5e 100644 --- a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/service/ConsumedMessageService.java +++ b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/service/ConsumedMessageService.java @@ -21,9 +21,11 @@ import com.github.sonus21.rqueue.test.dto.BaseQueueMessage; import com.github.sonus21.rqueue.test.entity.ConsumedMessage; import com.github.sonus21.rqueue.test.repository.ConsumedMessageRepository; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import lombok.NonNull; import lombok.RequiredArgsConstructor; @@ -33,27 +35,29 @@ @Service @RequiredArgsConstructor(onConstructor = @__(@Autowired)) public class ConsumedMessageService { - @NonNull private ConsumedMessageRepository consumedMessageRepository; - @NonNull private ObjectMapper objectMapper; + @NonNull private final ConsumedMessageRepository consumedMessageRepository; + @NonNull private final ObjectMapper objectMapper; - public ConsumedMessage save(BaseQueueMessage message, String tag) throws JsonProcessingException { + public ConsumedMessage save(BaseQueueMessage message, String tag, String queueName) + throws JsonProcessingException { String textMessage = objectMapper.writeValueAsString(message); - ConsumedMessage consumedMessage = new ConsumedMessage(message.getId(), textMessage, tag); + ConsumedMessage consumedMessage = + new ConsumedMessage(message.getId(), tag, queueName, textMessage); consumedMessageRepository.save(consumedMessage); return consumedMessage; } - public Collection getConsumedMessages(Collection ids) { - return getMessages(ids).values(); + public Collection getConsumedMessages(Collection messageIds) { + return getMessages(messageIds).values(); } - public T getMessage(String id, Class tClass) { - return getMessages(Collections.singletonList(id), tClass).get(id); + public T getMessage(String messageId, Class tClass) { + return getMessages(Collections.singletonList(messageId), tClass).get(messageId); } - public Map getMessages(Collection ids, Class tClass) { + public Map getMessages(Collection messageIds, Class tClass) { Map idToMessage = new HashMap<>(); - getMessages(ids) + getMessages(messageIds) .values() .forEach( consumedMessage -> { @@ -67,15 +71,31 @@ public Map getMessages(Collection ids, Class tClass) { return idToMessage; } - public Map getMessages(Collection ids) { - Iterable consumedMessages = consumedMessageRepository.findAllById(ids); + public Map getMessages(Collection messageIds) { + Iterable consumedMessages = + consumedMessageRepository.findByMessageIdIn(messageIds); Map idToMessage = new HashMap<>(); consumedMessages.forEach( consumedMessage -> idToMessage.put(consumedMessage.getId(), consumedMessage)); return idToMessage; } - public ConsumedMessage getConsumedMessage(String id) { - return consumedMessageRepository.findById(id).orElse(null); + public ConsumedMessage getConsumedMessage(String messageId) { + List messages = getConsumedMessages(messageId); + if (messages.size() == 0) { + return null; + } + if (messages.size() == 1) { + return messages.get(0); + } + throw new IllegalStateException("more than one record found"); + } + + public List getConsumedMessages(String messageId) { + return new ArrayList<>(consumedMessageRepository.findByMessageId(messageId)); + } + + public List getConsumedMessagesForQueue(String queueName) { + return new ArrayList<>(consumedMessageRepository.findByQueueName(queueName)); } } diff --git a/rqueue-common-test/src/main/resources/application.properties b/rqueue-common-test/src/main/resources/application.properties index e3bf246b..81948a43 100644 --- a/rqueue-common-test/src/main/resources/application.properties +++ b/rqueue-common-test/src/main/resources/application.properties @@ -41,5 +41,9 @@ reservation.request.active=false reservation.request.dead.letter.queue.retry.count=1 list.email.queue.name=email-list-queue list.email.queue.enabled=false +periodic.job.queue.name=periodic-job-queue +periodic.job.dead.letter.queue.name=periodic-job-dlq +periodic.job.queue.retry.count=3 +periodic.job.queue.active=false diff --git a/rqueue-common-test/src/main/resources/logback.xml b/rqueue-common-test/src/main/resources/logback.xml index 1b7b7617..c6d4fcb9 100644 --- a/rqueue-common-test/src/main/resources/logback.xml +++ b/rqueue-common-test/src/main/resources/logback.xml @@ -28,7 +28,7 @@ - + diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RedisScriptFactory.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RedisScriptFactory.java index 594ee4c9..0086809e 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RedisScriptFactory.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RedisScriptFactory.java @@ -38,6 +38,7 @@ public static RedisScript getScript(ScriptType type) { case MOVE_MESSAGE_LIST_TO_ZSET: case MOVE_MESSAGE_ZSET_TO_ZSET: case MOVE_MESSAGE_ZSET_TO_LIST: + case SCHEDULE_MESSAGE: script.setResultType(Long.class); return script; case DEQUEUE_MESSAGE: @@ -56,7 +57,8 @@ public enum ScriptType { MOVE_MESSAGE_LIST_TO_LIST("scripts/move_message_list_to_list.lua"), MOVE_MESSAGE_LIST_TO_ZSET("scripts/move_message_list_to_zset.lua"), MOVE_MESSAGE_ZSET_TO_ZSET("scripts/move_message_zset_to_zset.lua"), - MOVE_MESSAGE_ZSET_TO_LIST("scripts/move_message_zset_to_list.lua"); + MOVE_MESSAGE_ZSET_TO_LIST("scripts/move_message_zset_to_list.lua"), + SCHEDULE_MESSAGE("scripts/schedule_message.lua"); private final String path; ScriptType(String path) { diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessage.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessage.java index 0ea6c5b2..24269499 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessage.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessage.java @@ -32,8 +32,8 @@ @NoArgsConstructor @AllArgsConstructor @JsonPropertyOrder({"failureCount"}) -@Builder -public class RqueueMessage extends SerializableBase implements Cloneable { +@Builder(toBuilder = true) +public class RqueueMessage extends SerializableBase { private static final long serialVersionUID = -3488860960637488519L; // The message id, each message has a unique id @@ -60,12 +60,6 @@ public void updateReEnqueuedAt() { reEnqueuedAt = System.currentTimeMillis(); } - @Override - @SuppressWarnings("squid:S2975") - public RqueueMessage clone() throws CloneNotSupportedException { - return (RqueueMessage) super.clone(); - } - @Override public boolean equals(Object other) { if (other instanceof RqueueMessage) { @@ -77,6 +71,19 @@ public boolean equals(Object other) { return false; } + @JsonIgnore + public String getPseudoId() { + if (!isPeriodicTask()) { + return id; + } + return id + "::sch::" + processAt; + } + + @JsonIgnore + public long nextProcessAt() { + return processAt + period; + } + @JsonIgnore public boolean isPeriodicTask() { return period > 0; diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageEnqueuer.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageEnqueuer.java index d588177a..3aa4549f 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageEnqueuer.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageEnqueuer.java @@ -612,6 +612,33 @@ default boolean enqueueUniqueAtWithPriority( */ String enqueuePeriodic(String queueName, Object message, long periodInMilliSeconds); + /** + * Enqueue a message on given queue that will be running after a given period. It works like + * periodic cron that's scheduled at certain interval, for example every 30 seconds. + * + * @param queueName on which queue message has to be send + * @param message message object it could be any arbitrary object. + * @param period period of this job + * @param unit period unit + * @return message id on successful enqueue otherwise null. + */ + default String enqueuePeriodic(String queueName, Object message, long period, TimeUnit unit) { + return enqueuePeriodic(queueName, message, unit.toMillis(period)); + } + + /** + * Enqueue a message on given queue that will be running after a given period. It works like + * periodic cron that's scheduled at certain interval, for example every 30 seconds. + * + * @param queueName on which queue message has to be send + * @param message message object it could be any arbitrary object. + * @param period job period + * @return message id on successful enqueue otherwise null. + */ + default String enqueuePeriodic(String queueName, Object message, Duration period) { + return enqueuePeriodic(queueName, message, period.toMillis()); + } + /** * Enqueue a message on given queue that will be running after a given period. It works like * periodic cron that's scheduled at certain interval, for example every 30 seconds. @@ -620,8 +647,39 @@ default boolean enqueueUniqueAtWithPriority( * @param messageId message id corresponding to this message * @param message message object it could be any arbitrary object. * @param periodInMilliSeconds period of this job in milliseconds. - * @return message id on successful enqueue otherwise null. + * @return success or failure */ boolean enqueuePeriodic( String queueName, String messageId, Object message, long periodInMilliSeconds); + + /** + * Enqueue a message on given queue that will be running after a given period. It works like + * periodic cron that's scheduled at certain interval, for example every 30 seconds. + * + * @param queueName on which queue message has to be send + * @param messageId message id corresponding to this message + * @param message message object it could be any arbitrary object. + * @param period period of this job . + * @param unit unit of this period + * @return success or failure + */ + default boolean enqueuePeriodic( + String queueName, String messageId, Object message, long period, TimeUnit unit) { + return enqueuePeriodic(queueName, messageId, message, unit.toMillis(period)); + } + + /** + * Enqueue a message on given queue that will be running after a given period. It works like + * periodic cron that's scheduled at certain interval, for example every 30 seconds. + * + * @param queueName on which queue message has to be send + * @param messageId message id corresponding to this message + * @param message message object it could be any arbitrary object. + * @param period period of this job . + * @return success or failure + */ + default boolean enqueuePeriodic( + String queueName, String messageId, Object message, Duration period) { + return enqueuePeriodic(queueName, messageId, message, period.toMillis()); + } } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageTemplate.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageTemplate.java index a21fd47d..99c7fda6 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageTemplate.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageTemplate.java @@ -64,4 +64,6 @@ MessageMoveResult moveMessageZsetToZset( List> readFromZsetWithScore(String name, long start, long end); Long getScore(String delayedQueueName, RqueueMessage rqueueMessage); + + void scheduleMessage(String queueName, RqueueMessage rqueueMessage, long expiryInMilliSeconds); } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageTemplateImpl.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageTemplateImpl.java index 700216fd..733b4b7e 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageTemplateImpl.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageTemplateImpl.java @@ -220,6 +220,18 @@ public Long getScore(String delayedQueueName, RqueueMessage rqueueMessage) { return score.longValue(); } + @Override + public void scheduleMessage( + String zsetName, RqueueMessage rqueueMessage, long expiryInMilliSeconds) { + RedisScript script = (RedisScript) getScript(ScriptType.SCHEDULE_MESSAGE); + scriptExecutor.execute( + script, + Arrays.asList(rqueueMessage.getPseudoId(), zsetName), + expiryInMilliSeconds, + rqueueMessage, + rqueueMessage.getProcessAt()); + } + @Override public List readFromList(String name, long start, long end) { List messages = lrange(name, start, end); diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/PostProcessingHandler.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/PostProcessingHandler.java index 5effad7b..8e09f411 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/PostProcessingHandler.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/PostProcessingHandler.java @@ -133,14 +133,12 @@ void handle( } private void handleOldMessage(QueueDetail queueDetail, RqueueMessage rqueueMessage) { - if (isDebugEnabled()) { - log( - Level.DEBUG, - "Message {} ignored due to new message, Queue: {}", - null, - rqueueMessage, - queueDetail.getName()); - } + log( + Level.DEBUG, + "Message {} ignored due to new message, Queue: {}", + null, + rqueueMessage, + queueDetail.getName()); rqueueMessageTemplate.removeElementFromZset( queueDetail.getProcessingQueueName(), rqueueMessage); } @@ -206,7 +204,9 @@ private void moveMessageToQueue( byte[] processingQueueNameBytes = keySerializer.serialize(queueDetail.getProcessingQueueName()); byte[] queueNameBytes = keySerializer.serialize(queueName); + assert queueNameBytes != null; connection.rPush(queueNameBytes, newMessageBytes); + assert processingQueueNameBytes != null; connection.zRem(processingQueueNameBytes, oldMessageBytes); }); } @@ -242,18 +242,14 @@ private void moveMessageToDlq( Object userMessage, MessageMetadata messageMetadata, int failureCount, - long jobExecutionStartTime) - throws CloneNotSupportedException { - if (isWarningEnabled()) { - log( - Level.WARN, - "Message {} Moved to dead letter queue: {}", - null, - userMessage, - queueDetail.getDeadLetterQueueName()); - } - RqueueMessage newMessage = rqueueMessage.clone(); - newMessage.setFailureCount(failureCount); + long jobExecutionStartTime) { + log( + Level.WARN, + "Message {} Moved to dead letter queue: {}", + null, + userMessage, + queueDetail.getDeadLetterQueueName()); + RqueueMessage newMessage = rqueueMessage.toBuilder().failureCount(failureCount).build(); newMessage.updateReEnqueuedAt(); moveMessageForReprocessingOrDlq(queueDetail, rqueueMessage, newMessage, userMessage); publishEvent( @@ -271,13 +267,9 @@ private void parkMessageForRetry( MessageMetadata messageMetadata, int failureCount, long jobExecutionStartTime, - long delay) - throws CloneNotSupportedException { - if (isDebugEnabled()) { - log(Level.DEBUG, "Message {} will be retried in {}Ms", null, userMessage, delay); - } - RqueueMessage newMessage = rqueueMessage.clone(); - newMessage.setFailureCount(failureCount); + long delay) { + log(Level.DEBUG, "Message {} will be retried in {}Ms", null, userMessage, delay); + RqueueMessage newMessage = rqueueMessage.toBuilder().failureCount(failureCount).build(); newMessage.updateReEnqueuedAt(); rqueueMessageTemplate.moveMessage( queueDetail.getProcessingQueueName(), @@ -295,9 +287,7 @@ private void discardMessage( MessageMetadata messageMetadata, int failureCount, long jobExecutionStartTime) { - if (isDebugEnabled()) { - log(Level.DEBUG, "Message {} discarded due to retry limit exhaust", null, userMessage); - } + log(Level.DEBUG, "Message {} discarded due to retry limit exhaust", null, userMessage); deleteMessage( queueDetail, rqueueMessage, @@ -315,9 +305,7 @@ private void handleManualDeletion( MessageMetadata messageMetadata, int failureCount, long jobExecutionStartTime) { - if (isDebugEnabled()) { - log(Level.DEBUG, "Message Deleted {} successfully", null, rqueueMessage); - } + log(Level.DEBUG, "Message Deleted {} successfully", null, rqueueMessage); deleteMessage( queueDetail, rqueueMessage, @@ -335,9 +323,7 @@ private void handleSuccessFullExecution( MessageMetadata messageMetadata, int failureCount, long jobExecutionStartTime) { - if (isDebugEnabled()) { - log(Level.DEBUG, "Message consumed {} successfully", null, rqueueMessage); - } + log(Level.DEBUG, "Message consumed {} successfully", null, rqueueMessage); deleteMessage( queueDetail, rqueueMessage, @@ -354,8 +340,7 @@ private void handleRetryExceededMessage( Object userMessage, MessageMetadata messageMetadata, int failureCount, - long jobExecutionStartTime) - throws CloneNotSupportedException { + long jobExecutionStartTime) { if (queueDetail.isDlqSet()) { moveMessageToDlq( queueDetail, @@ -387,8 +372,7 @@ private void handleFailure( Object userMessage, MessageMetadata messageMetadata, int failureCount, - long jobExecutionStartTime) - throws CloneNotSupportedException { + long jobExecutionStartTime) { int maxRetryCount = getMaxRetryCount(rqueueMessage, queueDetail); if (failureCount < maxRetryCount) { long delay = taskExecutionBackoff.nextBackOff(userMessage, rqueueMessage, failureCount); @@ -428,9 +412,7 @@ private void handleIgnoredMessage( MessageMetadata messageMetadata, int failureCount, long jobExecutionStartTime) { - if (isDebugEnabled()) { - log(Level.DEBUG, "Message {} ignored, Queue: {}", null, rqueueMessage, queueDetail.getName()); - } + log(Level.DEBUG, "Message {} ignored, Queue: {}", null, rqueueMessage, queueDetail.getName()); deleteMessage( queueDetail, rqueueMessage, diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueExecutor.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueExecutor.java index 4f44d891..c5ab6161 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueExecutor.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueExecutor.java @@ -17,6 +17,7 @@ package com.github.sonus21.rqueue.listener; import static com.github.sonus21.rqueue.utils.Constants.DELTA_BETWEEN_RE_ENQUEUE_TIME; +import static com.github.sonus21.rqueue.utils.Constants.SECONDS_IN_A_DAY; import com.github.sonus21.rqueue.config.RqueueConfig; import com.github.sonus21.rqueue.core.RqueueMessage; @@ -185,8 +186,7 @@ private void logExecutionTimeWarning( } } - @Override - void start() { + private void processSimpleMessage() { int failureCount = rqueueMessage.getFailureCount(); long maxProcessingTime = getMaxProcessingTime(); long startTime = System.currentTimeMillis(); @@ -229,4 +229,21 @@ void start() { semaphore.release(); } } + + private void processScheduledMessage() { + RqueueMessage newMessage = + rqueueMessage.toBuilder().processAt(rqueueMessage.nextProcessAt()).build(); + getRqueueMessageTemplate() + .scheduleMessage(queueDetail.getDelayedQueueName(), newMessage, SECONDS_IN_A_DAY); + processSimpleMessage(); + } + + @Override + void start() { + if (rqueueMessage.isPeriodicTask()) { + processScheduledMessage(); + } else { + processSimpleMessage(); + } + } } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/ValueResolver.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/ValueResolver.java index b7dd5b14..fd2bd7dc 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/ValueResolver.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/ValueResolver.java @@ -62,7 +62,9 @@ public static Integer parseStringToInt(String val) { public static boolean convertToBoolean(String s) { String tmpString = clean(s); - if (tmpString.equalsIgnoreCase("true")) { + if (tmpString.equalsIgnoreCase("true") + || tmpString.equals("1") + || tmpString.equalsIgnoreCase("yes")) { return true; } if (tmpString.equalsIgnoreCase("false")) { diff --git a/rqueue-core/src/main/resources/scripts/schedule_message.lua b/rqueue-core/src/main/resources/scripts/schedule_message.lua new file mode 100644 index 00000000..a22b0fe6 --- /dev/null +++ b/rqueue-core/src/main/resources/scripts/schedule_message.lua @@ -0,0 +1,11 @@ +-- get current value +local value = redis.call('GET', KEYS[1]) + +if value == nil then + redis.call('SET', KEYS[1], "1", "EX", ARGV[1]) + redis.call('ZADD', KEYS[2], ARGV[3], ARGV[2]) + return 1 +end + +return 0 + diff --git a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/MessageSchedulingTest.java b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/MessageSchedulingTest.java new file mode 100644 index 00000000..aa4b7036 --- /dev/null +++ b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/MessageSchedulingTest.java @@ -0,0 +1,21 @@ +/* + * Copyright 2020 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.sonus21.rqueue.spring.boot.tests.integration; + +public class MessageSchedulingTest { + +} diff --git a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/PeriodicMessageTest.java b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/PeriodicMessageTest.java new file mode 100644 index 00000000..e02d075e --- /dev/null +++ b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/PeriodicMessageTest.java @@ -0,0 +1,82 @@ +/* + * Copyright 2020 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.sonus21.rqueue.spring.boot.tests.integration; + +import com.github.sonus21.junit.SpringTestTracerExtension; +import com.github.sonus21.rqueue.exception.TimedOutException; +import com.github.sonus21.rqueue.spring.boot.application.ApplicationWithCustomConfiguration; +import com.github.sonus21.rqueue.test.common.SpringTestBase; +import com.github.sonus21.rqueue.test.dto.PeriodicJob; +import com.github.sonus21.rqueue.utils.TimeoutUtils; +import java.time.Duration; +import java.util.concurrent.TimeUnit; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.TestPropertySource; + +@SpringBootTest +@ExtendWith(SpringTestTracerExtension.class) +@ContextConfiguration(classes = ApplicationWithCustomConfiguration.class) +@Slf4j +@TestPropertySource( + properties = { + "spring.redis.port=8013", + "mysql.db.name=PeriodicMessageTest", + "rqueue.metrics.count.failure=false", + "rqueue.metrics.count.execution=false", + "periodic.job.queue.active=true" + }) +public class PeriodicMessageTest extends SpringTestBase { + @Test + public void testSimplePeriodicMessage() throws TimedOutException { + PeriodicJob job = PeriodicJob.newInstance(); + String messageId = + rqueueMessageEnqueuer.enqueuePeriodic(periodicJobQueue, job, Duration.ofSeconds(2)); + TimeoutUtils.waitFor( + () -> { + printQueueStats(periodicJobQueue); + printConsumedMessage(periodicJobQueue); + return consumedMessageService.getConsumedMessages(job.getId()).size() > 1; + }, + "at least two execution"); + rqueueMessageManager.deleteMessage(periodicJobQueue, messageId); + } + + @Test + public void testPeriodicMessageWithTimeUnit() throws TimedOutException { + PeriodicJob job = PeriodicJob.newInstance(); + String messageId = + rqueueMessageEnqueuer.enqueuePeriodic(periodicJobQueue, job, 2000, TimeUnit.MILLISECONDS); + TimeoutUtils.waitFor( + () -> consumedMessageService.getConsumedMessages(job.getId()).size() > 1, + "at least two execution"); + rqueueMessageManager.deleteMessage(periodicJobQueue, messageId); + } + + @Test + public void testPeriodicMessageMilliseconds() throws TimedOutException { + PeriodicJob job = PeriodicJob.newInstance(); + String messageId = rqueueMessageEnqueuer.enqueuePeriodic(periodicJobQueue, job, 2000); + TimeoutUtils.waitFor( + () -> consumedMessageService.getConsumedMessages(job.getId()).size() > 1, + "at least two execution"); + rqueueMessageManager.deleteMessage(periodicJobQueue, messageId); + } +} diff --git a/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/services/ConsumedMessageRepositoryImpl.java b/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/services/ConsumedMessageRepositoryImpl.java index 3c2933c0..268c37aa 100644 --- a/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/services/ConsumedMessageRepositoryImpl.java +++ b/rqueue-spring/src/test/java/com/github/sonus21/rqueue/spring/services/ConsumedMessageRepositoryImpl.java @@ -18,6 +18,8 @@ import com.github.sonus21.rqueue.test.entity.ConsumedMessage; import com.github.sonus21.rqueue.test.repository.ConsumedMessageRepository; +import java.util.Collection; +import java.util.List; import java.util.Optional; import javax.persistence.EntityManager; import javax.persistence.EntityManagerFactory; @@ -82,4 +84,24 @@ public void deleteAll(Iterable entities) {} @Override public void deleteAll() {} + + @Override + public List findByQueueName(String queueName) { + return null; + } + + @Override + public List findByMessageId(String messageId) { + return null; + } + + @Override + public List findByMessageIdIn(Collection messageIds) { + return null; + } + + @Override + public ConsumedMessage findByMessageIdAndTag(String messageId, String tag) { + return null; + } } From a5e11c15c59fd5945928d6d0d4a6d64f0f3b7e97 Mon Sep 17 00:00:00 2001 From: Sonu Kumar Date: Thu, 3 Dec 2020 20:35:23 +0530 Subject: [PATCH 11/23] Scheduled message fix --- .../rqueue/test/entity/ConsumedMessage.java | 35 ++++++++++++++----- .../test/service/ConsumedMessageService.java | 14 +++++--- .../sonus21/rqueue/core/RqueueMessage.java | 8 ----- .../rqueue/core/RqueueMessageTemplate.java | 3 +- .../core/impl/RqueueMessageTemplateImpl.java | 8 ++--- .../rqueue/listener/RqueueExecutor.java | 15 +++++--- .../sonus21/rqueue/utils/TimeoutUtils.java | 6 +++- .../resources/scripts/schedule_message.lua | 10 +++--- .../integration/MessageSchedulingTest.java | 21 ----------- .../integration/PeriodicMessageTest.java | 5 ++- 10 files changed, 67 insertions(+), 58 deletions(-) delete mode 100644 rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/MessageSchedulingTest.java diff --git a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/entity/ConsumedMessage.java b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/entity/ConsumedMessage.java index 21825469..e99624d5 100644 --- a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/entity/ConsumedMessage.java +++ b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/entity/ConsumedMessage.java @@ -16,10 +16,11 @@ package com.github.sonus21.rqueue.test.entity; +import java.util.UUID; import javax.persistence.Column; import javax.persistence.Entity; -import javax.persistence.GeneratedValue; import javax.persistence.Id; +import javax.persistence.PreUpdate; import javax.persistence.Table; import javax.persistence.UniqueConstraint; import lombok.AllArgsConstructor; @@ -28,7 +29,6 @@ import lombok.NoArgsConstructor; import lombok.Setter; import lombok.ToString; -import org.hibernate.annotations.GenericGenerator; @AllArgsConstructor @NoArgsConstructor @@ -42,11 +42,7 @@ uniqueConstraints = {@UniqueConstraint(columnNames = {"message_id", "tag"})}) public class ConsumedMessage { - @Id - @GeneratedValue(generator = "UUID") - @GenericGenerator(name = "UUID", strategy = "org.hibernate.id.UUIDGenerator") - @Column - private String id; + @Id @Column private String id; @Column(name = "message_id") private String messageId; @@ -54,7 +50,7 @@ public class ConsumedMessage { @Column(name = "tag") private String tag; - private String queueName; + @Column private String queueName; // Around 1 MB of data @Column(length = 1000000) @@ -62,7 +58,28 @@ public class ConsumedMessage { @Column private Long createdAt; + @Column private Long updatedAt; + + @Column private int count; + + @PreUpdate + public void update() { + this.updatedAt = System.currentTimeMillis(); + } + public ConsumedMessage(String messageId, String tag, String queueName, String message) { - this(null, messageId, tag, queueName, message, System.currentTimeMillis()); + this( + UUID.randomUUID().toString(), + messageId, + tag, + queueName, + message, + System.currentTimeMillis(), + System.currentTimeMillis(), + 1); + } + + public void incrementCount() { + this.count += 1; } } diff --git a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/service/ConsumedMessageService.java b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/service/ConsumedMessageService.java index d8718a5e..a4a967f6 100644 --- a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/service/ConsumedMessageService.java +++ b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/service/ConsumedMessageService.java @@ -40,9 +40,15 @@ public class ConsumedMessageService { public ConsumedMessage save(BaseQueueMessage message, String tag, String queueName) throws JsonProcessingException { - String textMessage = objectMapper.writeValueAsString(message); ConsumedMessage consumedMessage = - new ConsumedMessage(message.getId(), tag, queueName, textMessage); + consumedMessageRepository.findByMessageIdAndTag(message.getId(), tag); + String textMessage = objectMapper.writeValueAsString(message); + if (consumedMessage == null) { + consumedMessage = new ConsumedMessage(message.getId(), tag, queueName, textMessage); + } else { + consumedMessage.incrementCount(); + consumedMessage.setMessage(textMessage); + } consumedMessageRepository.save(consumedMessage); return consumedMessage; } @@ -63,7 +69,7 @@ public Map getMessages(Collection messageIds, Class tC consumedMessage -> { try { T value = objectMapper.readValue(consumedMessage.getMessage(), tClass); - idToMessage.put(consumedMessage.getId(), value); + idToMessage.put(consumedMessage.getMessageId(), value); } catch (JsonProcessingException e) { e.printStackTrace(); } @@ -76,7 +82,7 @@ public Map getMessages(Collection messageIds) { consumedMessageRepository.findByMessageIdIn(messageIds); Map idToMessage = new HashMap<>(); consumedMessages.forEach( - consumedMessage -> idToMessage.put(consumedMessage.getId(), consumedMessage)); + consumedMessage -> idToMessage.put(consumedMessage.getMessageId(), consumedMessage)); return idToMessage; } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessage.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessage.java index 24269499..3e0ff7b3 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessage.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessage.java @@ -71,14 +71,6 @@ public boolean equals(Object other) { return false; } - @JsonIgnore - public String getPseudoId() { - if (!isPeriodicTask()) { - return id; - } - return id + "::sch::" + processAt; - } - @JsonIgnore public long nextProcessAt() { return processAt + period; diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageTemplate.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageTemplate.java index 99c7fda6..a330c268 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageTemplate.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/RqueueMessageTemplate.java @@ -65,5 +65,6 @@ MessageMoveResult moveMessageZsetToZset( Long getScore(String delayedQueueName, RqueueMessage rqueueMessage); - void scheduleMessage(String queueName, RqueueMessage rqueueMessage, long expiryInMilliSeconds); + Long scheduleMessage( + String queueName, String messageId, RqueueMessage rqueueMessage, long expiryInMilliSeconds); } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageTemplateImpl.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageTemplateImpl.java index 733b4b7e..56817288 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageTemplateImpl.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageTemplateImpl.java @@ -221,12 +221,12 @@ public Long getScore(String delayedQueueName, RqueueMessage rqueueMessage) { } @Override - public void scheduleMessage( - String zsetName, RqueueMessage rqueueMessage, long expiryInMilliSeconds) { + public Long scheduleMessage( + String zsetName, String messageId, RqueueMessage rqueueMessage, long expiryInMilliSeconds) { RedisScript script = (RedisScript) getScript(ScriptType.SCHEDULE_MESSAGE); - scriptExecutor.execute( + return scriptExecutor.execute( script, - Arrays.asList(rqueueMessage.getPseudoId(), zsetName), + Arrays.asList(messageId, zsetName), expiryInMilliSeconds, rqueueMessage, rqueueMessage.getProcessAt()); diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueExecutor.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueExecutor.java index c5ab6161..98bf2da8 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueExecutor.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueExecutor.java @@ -230,18 +230,25 @@ private void processSimpleMessage() { } } - private void processScheduledMessage() { + private void processPeriodicMessage() { RqueueMessage newMessage = rqueueMessage.toBuilder().processAt(rqueueMessage.nextProcessAt()).build(); - getRqueueMessageTemplate() - .scheduleMessage(queueDetail.getDelayedQueueName(), newMessage, SECONDS_IN_A_DAY); + // avoid duplicate message enqueue due to retry by checking the message key + String messageId = + rqueueConfig.getPrefix() + rqueueMessage.getId() + "::sch::" + newMessage.getProcessAt(); + log.info( + "Schedule periodic message: {} Status: {}", + rqueueMessage, + getRqueueMessageTemplate() + .scheduleMessage( + queueDetail.getDelayedQueueName(), messageId, newMessage, SECONDS_IN_A_DAY)); processSimpleMessage(); } @Override void start() { if (rqueueMessage.isPeriodicTask()) { - processScheduledMessage(); + processPeriodicMessage(); } else { processSimpleMessage(); } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/TimeoutUtils.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/TimeoutUtils.java index 821327fb..a0aa32ab 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/TimeoutUtils.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/TimeoutUtils.java @@ -55,11 +55,15 @@ public static void waitFor( Runnable postmortem) throws TimedOutException { long endTime = System.currentTimeMillis() + waitTimeInMilliSeconds; + long sleepTime = 100L; + if (waitTimeInMilliSeconds > 2000) { + sleepTime = 200; + } do { if (Boolean.TRUE.equals(callback.getAsBoolean())) { return; } - sleep(100L); + sleep(sleepTime); } while (System.currentTimeMillis() < endTime); try { postmortem.run(); diff --git a/rqueue-core/src/main/resources/scripts/schedule_message.lua b/rqueue-core/src/main/resources/scripts/schedule_message.lua index a22b0fe6..097e3faf 100644 --- a/rqueue-core/src/main/resources/scripts/schedule_message.lua +++ b/rqueue-core/src/main/resources/scripts/schedule_message.lua @@ -1,11 +1,11 @@ -- get current value local value = redis.call('GET', KEYS[1]) -if value == nil then - redis.call('SET', KEYS[1], "1", "EX", ARGV[1]) - redis.call('ZADD', KEYS[2], ARGV[3], ARGV[2]) - return 1 +if value then + return 0 end -return 0 +redis.call('SET', KEYS[1], "1", "EX", ARGV[1]) +redis.call('ZADD', KEYS[2], ARGV[3], ARGV[2]) +return 1 diff --git a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/MessageSchedulingTest.java b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/MessageSchedulingTest.java deleted file mode 100644 index aa4b7036..00000000 --- a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/MessageSchedulingTest.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright 2020 Sonu Kumar - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.github.sonus21.rqueue.spring.boot.tests.integration; - -public class MessageSchedulingTest { - -} diff --git a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/PeriodicMessageTest.java b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/PeriodicMessageTest.java index e02d075e..4ebcc72b 100644 --- a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/PeriodicMessageTest.java +++ b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/PeriodicMessageTest.java @@ -41,7 +41,9 @@ "mysql.db.name=PeriodicMessageTest", "rqueue.metrics.count.failure=false", "rqueue.metrics.count.execution=false", - "periodic.job.queue.active=true" + "periodic.job.queue.active=true", + "use.system.redis=true", + "spring.redis.port=6379", }) public class PeriodicMessageTest extends SpringTestBase { @Test @@ -55,6 +57,7 @@ public void testSimplePeriodicMessage() throws TimedOutException { printConsumedMessage(periodicJobQueue); return consumedMessageService.getConsumedMessages(job.getId()).size() > 1; }, + 30_000, "at least two execution"); rqueueMessageManager.deleteMessage(periodicJobQueue, messageId); } From b7f7bac00a839f52443216e55f027facc655d72a Mon Sep 17 00:00:00 2001 From: Sonu Kumar Date: Fri, 4 Dec 2020 22:07:24 +0530 Subject: [PATCH 12/23] do not use system redis --- .../spring/boot/tests/integration/PeriodicMessageTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/PeriodicMessageTest.java b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/PeriodicMessageTest.java index 4ebcc72b..2078d54d 100644 --- a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/PeriodicMessageTest.java +++ b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/PeriodicMessageTest.java @@ -42,8 +42,7 @@ "rqueue.metrics.count.failure=false", "rqueue.metrics.count.execution=false", "periodic.job.queue.active=true", - "use.system.redis=true", - "spring.redis.port=6379", + "use.system.redis=false", }) public class PeriodicMessageTest extends SpringTestBase { @Test From 9e16920d1a1e311c9f452d31da4a71e96b9d797b Mon Sep 17 00:00:00 2001 From: Sonu Kumar Date: Fri, 4 Dec 2020 22:15:55 +0530 Subject: [PATCH 13/23] increase timeout --- .../spring/boot/tests/integration/PeriodicMessageTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/PeriodicMessageTest.java b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/PeriodicMessageTest.java index 2078d54d..c6c583d1 100644 --- a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/PeriodicMessageTest.java +++ b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/PeriodicMessageTest.java @@ -68,6 +68,7 @@ public void testPeriodicMessageWithTimeUnit() throws TimedOutException { rqueueMessageEnqueuer.enqueuePeriodic(periodicJobQueue, job, 2000, TimeUnit.MILLISECONDS); TimeoutUtils.waitFor( () -> consumedMessageService.getConsumedMessages(job.getId()).size() > 1, + 30_000, "at least two execution"); rqueueMessageManager.deleteMessage(periodicJobQueue, messageId); } @@ -78,6 +79,7 @@ public void testPeriodicMessageMilliseconds() throws TimedOutException { String messageId = rqueueMessageEnqueuer.enqueuePeriodic(periodicJobQueue, job, 2000); TimeoutUtils.waitFor( () -> consumedMessageService.getConsumedMessages(job.getId()).size() > 1, + 30_000, "at least two execution"); rqueueMessageManager.deleteMessage(periodicJobQueue, messageId); } From 61882c73b1344222fb9ca3ac9582ccb6371971ad Mon Sep 17 00:00:00 2001 From: Sonu Kumar Date: Fri, 4 Dec 2020 22:24:41 +0530 Subject: [PATCH 14/23] disable test in ci env --- .../spring/boot/tests/integration/PeriodicMessageTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/PeriodicMessageTest.java b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/PeriodicMessageTest.java index c6c583d1..f35f38b2 100644 --- a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/PeriodicMessageTest.java +++ b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/PeriodicMessageTest.java @@ -26,6 +26,7 @@ import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.DisabledIfEnvironmentVariable; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.context.ContextConfiguration; @@ -74,6 +75,7 @@ public void testPeriodicMessageWithTimeUnit() throws TimedOutException { } @Test + @DisabledIfEnvironmentVariable(named = "CI", matches = "true") public void testPeriodicMessageMilliseconds() throws TimedOutException { PeriodicJob job = PeriodicJob.newInstance(); String messageId = rqueueMessageEnqueuer.enqueuePeriodic(periodicJobQueue, job, 2000); From de5b5c3728ea7822e4b9ed722fdf7d96fe92d96b Mon Sep 17 00:00:00 2001 From: Sonu Kumar Date: Fri, 4 Dec 2020 22:26:11 +0530 Subject: [PATCH 15/23] Revert "disable test in ci env" This reverts commit 61882c73 --- .../spring/boot/tests/integration/PeriodicMessageTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/PeriodicMessageTest.java b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/PeriodicMessageTest.java index f35f38b2..c6c583d1 100644 --- a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/PeriodicMessageTest.java +++ b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/PeriodicMessageTest.java @@ -26,7 +26,6 @@ import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.condition.DisabledIfEnvironmentVariable; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.context.ContextConfiguration; @@ -75,7 +74,6 @@ public void testPeriodicMessageWithTimeUnit() throws TimedOutException { } @Test - @DisabledIfEnvironmentVariable(named = "CI", matches = "true") public void testPeriodicMessageMilliseconds() throws TimedOutException { PeriodicJob job = PeriodicJob.newInstance(); String messageId = rqueueMessageEnqueuer.enqueuePeriodic(periodicJobQueue, job, 2000); From 4eb54e51f3554b9f7b1f67b96bd3d496e3ba517b Mon Sep 17 00:00:00 2001 From: Sonu Kumar Date: Sat, 5 Dec 2020 12:11:19 +0530 Subject: [PATCH 16/23] Fixed potential cross slot issue with redis cluster in periodic job. --- .../rqueue/test/common/SpringTestBase.java | 8 +- .../core/DefaultRqueueMessageConverter.java | 6 + .../rqueue/core/impl/BaseMessageSender.java | 4 +- .../core/support/RqueueMessageUtils.java | 70 +++--- .../rqueue/listener/RqueueExecutor.java | 5 +- .../core/support/RqueueMessageUtilsTest.java | 228 ++++++++++++++++++ .../web/service/RqueueQDetailServiceTest.java | 6 +- 7 files changed, 276 insertions(+), 51 deletions(-) create mode 100644 rqueue-core/src/test/java/com/github/sonus21/rqueue/core/support/RqueueMessageUtilsTest.java diff --git a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/common/SpringTestBase.java b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/common/SpringTestBase.java index a36ccb74..b1354601 100644 --- a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/common/SpringTestBase.java +++ b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/common/SpringTestBase.java @@ -108,7 +108,7 @@ public abstract class SpringTestBase extends TestBase { protected void enqueue(Object message, String queueName) { RqueueMessage rqueueMessage = RqueueMessageUtils.buildMessage( - rqueueMessageManager.getMessageConverter(), message, queueName, null, null, null); + rqueueMessageManager.getMessageConverter(), queueName, message, null, null, null); rqueueMessageTemplate.addMessage(queueName, rqueueMessage); } @@ -117,7 +117,7 @@ protected void enqueue(String queueName, Factory factory, int n) { Object object = factory.next(i); RqueueMessage rqueueMessage = RqueueMessageUtils.buildMessage( - rqueueMessageManager.getMessageConverter(), object, queueName, null, null, null); + rqueueMessageManager.getMessageConverter(), queueName, object, null, null, null); rqueueMessageTemplate.addMessage(queueName, rqueueMessage); } } @@ -128,7 +128,7 @@ protected void enqueueIn(String zsetName, Factory factory, Delay delay, int n) { long score = delay.getDelay(i); RqueueMessage rqueueMessage = RqueueMessageUtils.buildMessage( - rqueueMessageManager.getMessageConverter(), object, zsetName, null, score, null); + rqueueMessageManager.getMessageConverter(), zsetName, object, null, score, null); rqueueMessageTemplate.addToZset(zsetName, rqueueMessage, rqueueMessage.getProcessAt()); } } @@ -136,7 +136,7 @@ protected void enqueueIn(String zsetName, Factory factory, Delay delay, int n) { protected void enqueueIn(Object message, String zsetName, long delay) { RqueueMessage rqueueMessage = RqueueMessageUtils.buildMessage( - rqueueMessageManager.getMessageConverter(), message, zsetName, null, delay, null); + rqueueMessageManager.getMessageConverter(), zsetName, message, null, delay, null); rqueueMessageTemplate.addToZset(zsetName, rqueueMessage, rqueueMessage.getProcessAt()); } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/DefaultRqueueMessageConverter.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/DefaultRqueueMessageConverter.java index a804e611..5fddef1f 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/DefaultRqueueMessageConverter.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/DefaultRqueueMessageConverter.java @@ -18,8 +18,10 @@ import com.github.sonus21.rqueue.converter.GenericMessageConverter; import com.google.common.collect.ImmutableList; +import java.util.Collection; import lombok.EqualsAndHashCode; import org.springframework.messaging.converter.CompositeMessageConverter; +import org.springframework.messaging.converter.MessageConverter; import org.springframework.messaging.converter.StringMessageConverter; @EqualsAndHashCode(callSuper = true) @@ -28,4 +30,8 @@ public final class DefaultRqueueMessageConverter extends CompositeMessageConvert public DefaultRqueueMessageConverter() { super(ImmutableList.of(new GenericMessageConverter(), new StringMessageConverter())); } + + public DefaultRqueueMessageConverter(Collection converters) { + super(converters); + } } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/BaseMessageSender.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/BaseMessageSender.java index 15bc88d4..ebced0d1 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/BaseMessageSender.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/BaseMessageSender.java @@ -19,8 +19,6 @@ import static com.github.sonus21.rqueue.core.support.RqueueMessageUtils.buildMessage; import static com.github.sonus21.rqueue.core.support.RqueueMessageUtils.buildPeriodicMessage; import static com.github.sonus21.rqueue.utils.Constants.MIN_DELAY; -import static com.github.sonus21.rqueue.utils.Validator.validateMessage; -import static com.github.sonus21.rqueue.utils.Validator.validatePeriod; import static com.github.sonus21.rqueue.utils.Validator.validateQueue; import static org.springframework.util.Assert.notNull; @@ -84,7 +82,7 @@ private RqueueMessage constructMessage( Long delayInMilliSecs) { RqueueMessage rqueueMessage = buildMessage( - messageConverter, message, queueName, retryCount, delayInMilliSecs, messageHeaders); + messageConverter, queueName, message, retryCount, delayInMilliSecs, messageHeaders); if (messageId != null) { rqueueMessage.setId(messageId); } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/support/RqueueMessageUtils.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/support/RqueueMessageUtils.java index a79a79a1..2d339cbf 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/support/RqueueMessageUtils.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/support/RqueueMessageUtils.java @@ -57,35 +57,32 @@ public static RqueueMessage buildPeriodicMessage( throw new MessageConversionException("Message could not be build (null)"); } Object payload = msg.getPayload(); + long processAt = System.currentTimeMillis() + period; + String strMessage; if (payload instanceof String) { - return RqueueMessage.builder() - .period(period) - .id(UUID.randomUUID().toString()) - .queueName(queueName) - .message((String) payload) - .processAt(System.currentTimeMillis() + period) - .build(); + strMessage = (String) payload; + } else if (payload instanceof byte[]) { + strMessage = new String((byte[]) payload); + } else { + throw new MessageConversionException("Message payload is neither String nor byte[]"); } - if (payload instanceof byte[]) { - return RqueueMessage.builder() - .period(period) - .id(UUID.randomUUID().toString()) - .queueName(queueName) - .message(new String((byte[]) msg.getPayload())) - .processAt(System.currentTimeMillis() + period) - .build(); - } - throw new MessageConversionException("Message payload is neither String nor byte[]"); + return RqueueMessage.builder() + .id(UUID.randomUUID().toString()) + .queueName(queueName) + .message(strMessage) + .processAt(processAt) + .period(period) + .build(); } public static RqueueMessage buildMessage( MessageConverter converter, - Object object, String queueName, + Object message, Integer retryCount, Long delay, MessageHeaders messageHeaders) { - Message msg = converter.toMessage(object, messageHeaders); + Message msg = converter.toMessage(message, messageHeaders); if (msg == null) { throw new MessageConversionException("Message could not be build (null)"); } @@ -95,27 +92,22 @@ public static RqueueMessage buildMessage( processAt += delay; } Object payload = msg.getPayload(); + String strMessage; if (payload instanceof String) { - return RqueueMessage.builder() - .retryCount(retryCount) - .queuedTime(queuedTime) - .id(UUID.randomUUID().toString()) - .queueName(queueName) - .message((String) payload) - .processAt(processAt) - .build(); - } - if (payload instanceof byte[]) { - return RqueueMessage.builder() - .retryCount(retryCount) - .queuedTime(queuedTime) - .id(UUID.randomUUID().toString()) - .queueName(queueName) - .message(new String((byte[]) msg.getPayload())) - .processAt(processAt) - .build(); + strMessage = (String) payload; + } else if (payload instanceof byte[]) { + strMessage = new String((byte[]) payload); + } else { + throw new MessageConversionException("Message payload is neither String nor byte[]"); } - throw new MessageConversionException("Message payload is neither String nor byte[]"); + return RqueueMessage.builder() + .retryCount(retryCount) + .queuedTime(queuedTime) + .id(UUID.randomUUID().toString()) + .queueName(queueName) + .message(strMessage) + .processAt(processAt) + .build(); } public static List generateMessages( @@ -137,7 +129,7 @@ public static List generateMessages( int count) { List messages = new ArrayList<>(); for (int i = 0; i < count; i++) { - messages.add(buildMessage(converter, object, queueName, retryCount, delay, null)); + messages.add(buildMessage(converter, queueName, object, retryCount, delay, null)); } return messages; } diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueExecutor.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueExecutor.java index 98bf2da8..1e516640 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueExecutor.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueExecutor.java @@ -234,9 +234,10 @@ private void processPeriodicMessage() { RqueueMessage newMessage = rqueueMessage.toBuilder().processAt(rqueueMessage.nextProcessAt()).build(); // avoid duplicate message enqueue due to retry by checking the message key + // avoid cross slot error by using tagged queue name in the key String messageId = - rqueueConfig.getPrefix() + rqueueMessage.getId() + "::sch::" + newMessage.getProcessAt(); - log.info( + queueDetail.getQueueName() + rqueueMessage.getId() + "::sch::" + newMessage.getProcessAt(); + log.debug( "Schedule periodic message: {} Status: {}", rqueueMessage, getRqueueMessageTemplate() diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/support/RqueueMessageUtilsTest.java b/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/support/RqueueMessageUtilsTest.java new file mode 100644 index 00000000..2511b780 --- /dev/null +++ b/rqueue-core/src/test/java/com/github/sonus21/rqueue/core/support/RqueueMessageUtilsTest.java @@ -0,0 +1,228 @@ +/* + * Copyright 2020 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.sonus21.rqueue.core.support; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +import com.github.sonus21.rqueue.converter.GenericMessageConverter; +import com.github.sonus21.rqueue.core.DefaultRqueueMessageConverter; +import com.github.sonus21.rqueue.core.RqueueMessage; +import com.github.sonus21.rqueue.listener.RqueueMessageHeaders; +import com.google.common.collect.ImmutableList; +import java.util.UUID; +import lombok.Data; +import org.apache.commons.lang3.RandomStringUtils; +import org.junit.jupiter.api.Test; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageHeaders; +import org.springframework.messaging.converter.MessageConversionException; +import org.springframework.messaging.converter.MessageConverter; +import org.springframework.messaging.converter.StringMessageConverter; +import org.springframework.messaging.support.GenericMessage; + +class RqueueMessageUtilsTest { + + DefaultRqueueMessageConverter messageConverter = new DefaultRqueueMessageConverter(); + private final String queue = "test-queue"; + DefaultRqueueMessageConverter messageConverter2 = + new DefaultRqueueMessageConverter( + ImmutableList.of( + new GenericMessageConverter(), + new StringMessageConverter(), + new NoMessageConverter())); + + static class NoMessageConverter implements MessageConverter { + + @Override + public Object fromMessage(Message message, Class aClass) { + return null; + } + + @Override + public Message toMessage(Object o, MessageHeaders messageHeaders) { + return new GenericMessage<>(o); + } + } + + @Data + static class Email { + String id; + String email; + + static Email newInstance() { + Email email = new Email(); + email.id = UUID.randomUUID().toString(); + email.email = RandomStringUtils.randomAlphabetic(10) + "@test.com"; + return email; + } + } + + @Data + static class GenericClass { + T id; + } + + @Test + void buildPeriodicMessage() { + Email email = Email.newInstance(); + long startTime = System.currentTimeMillis(); + RqueueMessage message = + RqueueMessageUtils.buildPeriodicMessage( + messageConverter, queue, email, 10_000L, RqueueMessageHeaders.emptyMessageHeaders()); + assertEquals(10_000L, message.getPeriod()); + long now = System.currentTimeMillis(); + assertTrue( + message.getProcessAt() >= startTime + 10_000L && message.getProcessAt() <= now + 10_000L); + assertEquals(queue, message.getQueueName()); + assertEquals(0, message.getQueuedTime()); + assertNotNull(message.getId()); + assertNotNull(message.getMessage()); + String convertedMessage = + (String) + messageConverter + .toMessage(email, RqueueMessageHeaders.emptyMessageHeaders(), null) + .getPayload(); + assertEquals(convertedMessage, message.getMessage()); + } + + @Test + void buildMessage() { + Email email = Email.newInstance(); + long startTime = System.currentTimeMillis(); + long startTimeInNano = System.nanoTime(); + RqueueMessage message = + RqueueMessageUtils.buildMessage( + messageConverter, queue, email, null, null, RqueueMessageHeaders.emptyMessageHeaders()); + assertEquals(0, message.getPeriod()); + long now = System.currentTimeMillis(); + long nowNano = System.nanoTime(); + assertTrue(message.getProcessAt() >= startTime && message.getProcessAt() <= now); + assertTrue(message.getQueuedTime() >= startTimeInNano && message.getQueuedTime() <= nowNano); + assertEquals(queue, message.getQueueName()); + assertNotNull(message.getId()); + assertNotNull(message.getMessage()); + assertNull(message.getRetryCount()); + assertEquals(0, message.getFailureCount()); + String convertedMessage = + (String) + messageConverter + .toMessage(email, RqueueMessageHeaders.emptyMessageHeaders(), null) + .getPayload(); + assertEquals(convertedMessage, message.getMessage()); + } + + @Test + void buildMessageWithDelay() { + Email email = Email.newInstance(); + long startTime = System.currentTimeMillis(); + long startTimeInNano = System.nanoTime(); + RqueueMessage message = + RqueueMessageUtils.buildMessage( + messageConverter, queue, email, 3, 10_000L, RqueueMessageHeaders.emptyMessageHeaders()); + assertEquals(0, message.getPeriod()); + long now = System.currentTimeMillis(); + long nowNano = System.nanoTime(); + assertTrue( + message.getProcessAt() >= startTime + 10_000L && message.getProcessAt() <= now + 10_000L); + assertTrue(message.getQueuedTime() >= startTimeInNano && message.getQueuedTime() <= nowNano); + assertEquals(queue, message.getQueueName()); + assertNotNull(message.getId()); + assertNotNull(message.getMessage()); + assertEquals(3, message.getRetryCount()); + assertEquals(0, message.getFailureCount()); + String convertedMessage = + (String) + messageConverter + .toMessage(email, RqueueMessageHeaders.emptyMessageHeaders(), null) + .getPayload(); + assertEquals(convertedMessage, message.getMessage()); + } + + @Test + void buildMessageNull() { + GenericClass genericClass = new GenericClass<>(); + genericClass.id = UUID.randomUUID().toString(); + try { + RqueueMessageUtils.buildMessage( + messageConverter, + queue, + genericClass, + 3, + 10_000L, + RqueueMessageHeaders.emptyMessageHeaders()); + fail("message conversion should fail"); + } catch (MessageConversionException e) { + assertEquals("Message could not be build (null)", e.getMessage()); + } + } + + @Test + void buildPeriodicMessageNull() { + GenericClass genericClass = new GenericClass<>(); + genericClass.id = UUID.randomUUID().toString(); + try { + RqueueMessageUtils.buildPeriodicMessage( + messageConverter, + queue, + genericClass, + 10_000L, + RqueueMessageHeaders.emptyMessageHeaders()); + fail("message conversion should fail"); + } catch (MessageConversionException e) { + assertEquals("Message could not be build (null)", e.getMessage()); + } + } + + @Test + void buildMessageReturnInvalidType() { + GenericClass genericClass = new GenericClass<>(); + genericClass.id = UUID.randomUUID().toString(); + try { + RqueueMessageUtils.buildMessage( + messageConverter2, + queue, + genericClass, + 3, + 10_000L, + RqueueMessageHeaders.emptyMessageHeaders()); + fail("message conversion should fail"); + } catch (MessageConversionException e) { + assertEquals("Message payload is neither String nor byte[]", e.getMessage()); + } + } + + @Test + void buildPeriodicMessageReturnInvalidType() { + GenericClass genericClass = new GenericClass<>(); + genericClass.id = UUID.randomUUID().toString(); + try { + RqueueMessageUtils.buildPeriodicMessage( + messageConverter2, + queue, + genericClass, + 10_000L, + RqueueMessageHeaders.emptyMessageHeaders()); + fail("message conversion should fail"); + } catch (MessageConversionException e) { + assertEquals("Message payload is neither String nor byte[]", e.getMessage()); + } + } +} diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/web/service/RqueueQDetailServiceTest.java b/rqueue-core/src/test/java/com/github/sonus21/rqueue/web/service/RqueueQDetailServiceTest.java index 19227437..77648be6 100644 --- a/rqueue-core/src/test/java/com/github/sonus21/rqueue/web/service/RqueueQDetailServiceTest.java +++ b/rqueue-core/src/test/java/com/github/sonus21/rqueue/web/service/RqueueQDetailServiceTest.java @@ -328,7 +328,7 @@ public void viewDataList() { objects.add("Test"); objects.add( RqueueMessageUtils.buildMessage( - messageConverter, "buildMessage", "jobs", null, null, null)); + messageConverter, "jobs", "buildMessage", null, null, null)); objects.add(null); doReturn(objects).when(stringRqueueRedisTemplate).lrange("jobs", 0, 9); DataViewResponse response = rqueueQDetailService.viewData("jobs", DataType.LIST, null, 0, 10); @@ -349,7 +349,7 @@ public void viewDataZset() { objects.add( new DefaultTypedTuple<>( RqueueMessageUtils.buildMessage( - messageConverter, "buildMessage", "jobs", null, null, null), + messageConverter, "jobs", "buildMessage", null, null, null), 200.0)); List> rows = new ArrayList<>(); @@ -378,7 +378,7 @@ public void viewDataSet() { Set objects = new HashSet<>(); objects.add("Test"); objects.add( - RqueueMessageUtils.buildMessage(messageConverter, "Test object", "jobs", null, null, null)); + RqueueMessageUtils.buildMessage(messageConverter, "jobs", "Test object", null, null, null)); List> rows = new ArrayList<>(); for (Object object : objects) { rows.add(Collections.singletonList(String.valueOf(object))); From 7c1d61760619cfdc043c3e791d29902931c734c7 Mon Sep 17 00:00:00 2001 From: Sonu Kumar Date: Sat, 5 Dec 2020 12:21:17 +0530 Subject: [PATCH 17/23] Generic check for list item. --- .../converter/GenericMessageConverter.java | 26 ++-- .../GenericMessageConverterTest.java | 126 +++++++++--------- 2 files changed, 73 insertions(+), 79 deletions(-) diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/GenericMessageConverter.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/GenericMessageConverter.java index 4215f89b..7f93e3d1 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/GenericMessageConverter.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/GenericMessageConverter.java @@ -37,29 +37,20 @@ /** * A converter to turn the payload of a {@link Message} from serialized form to a typed String and - * vice versa. + * vice versa. This class does not support generic class except {@link List},even for list the + * entries should be non generic. */ @Slf4j public class GenericMessageConverter implements MessageConverter { private final ObjectMapper objectMapper; - private final boolean genericFieldEnabled; public GenericMessageConverter() { this(new ObjectMapper()); } public GenericMessageConverter(ObjectMapper objectMapper) { - this(objectMapper, true); - } - - public GenericMessageConverter(boolean genericFieldEnabled) { - this(new ObjectMapper(), genericFieldEnabled); - } - - public GenericMessageConverter(ObjectMapper objectMapper, boolean genericFieldEnabled) { notNull(objectMapper, "objectMapper cannot be null"); this.objectMapper = objectMapper; - this.genericFieldEnabled = genericFieldEnabled; } /** @@ -109,15 +100,16 @@ private String getClassNameForCollection(String name, Collection payload) { if (payload.isEmpty()) { return null; } - return name + '#' + ((List) payload).get(0).getClass().getName(); + String itemClassName = getClassName(((List) payload).get(0)); + if (itemClassName == null) { + return null; + } + return name + '#' + itemClassName; } return null; } - private String getGenericFieldBasedClassName(Class clazz, Object payload) { - if (!genericFieldEnabled) { - return clazz.getName(); - } + private String getGenericFieldBasedClassName(Class clazz) { TypeVariable[] typeVariables = clazz.getTypeParameters(); if (typeVariables.length == 0) { return clazz.getName(); @@ -131,7 +123,7 @@ private String getClassName(Object payload) { if (payload instanceof Collection) { return getClassNameForCollection(name, (Collection) payload); } - return getGenericFieldBasedClassName(payloadClass, payload); + return getGenericFieldBasedClassName(payloadClass); } /** diff --git a/rqueue-core/src/test/java/com/github/sonus21/rqueue/converter/GenericMessageConverterTest.java b/rqueue-core/src/test/java/com/github/sonus21/rqueue/converter/GenericMessageConverterTest.java index 3c27d03e..1849c291 100644 --- a/rqueue-core/src/test/java/com/github/sonus21/rqueue/converter/GenericMessageConverterTest.java +++ b/rqueue-core/src/test/java/com/github/sonus21/rqueue/converter/GenericMessageConverterTest.java @@ -47,6 +47,66 @@ public class GenericMessageConverterTest { private static final GenericMessageConverter genericMessageConverter = new GenericMessageConverter(); + + @Data + @NoArgsConstructor + public static class MultiLevelGenericTestDataNoArgs { + private String data; + private GenericTestData tGenericTestData; + private GenericTestData vGenericTestData; + } + + @Data + @AllArgsConstructor + @NoArgsConstructor + public static class MultiLevelGenericTestData { + private String data; + private GenericTestData tGenericTestData; + private GenericTestData vGenericTestData; + } + + @Data + @AllArgsConstructor + @NoArgsConstructor + public static class MultiLevelGenericTestDataFixedType { + private String data; + private GenericTestData tGenericTestData; + private MultiGenericTestData vGenericTestData; + } + + @Data + @AllArgsConstructor + @NoArgsConstructor + public static class MultiGenericTestData { + private Integer index; + private K key; + private V value; + } + + @Data + @AllArgsConstructor + @NoArgsConstructor + public static class GenericTestData { + private Integer index; + private T data; + } + + @Data + @AllArgsConstructor + @NoArgsConstructor + public static class Comment { + private String id; + private String message; + } + + @Data + @AllArgsConstructor + @NoArgsConstructor + public static class Email { + private String id; + private String subject; + } + private static Comment comment = new Comment(UUID.randomUUID().toString(), "This is test"); private static Email email = new Email(UUID.randomUUID().toString(), "This is test"); @@ -98,10 +158,11 @@ public void toMessageEmptyList() { } @Test - public void testMessageNonEmptyList() { + public void testMessageNonEmptyListWithGenericItem() { + List> items = + Collections.singletonList(new GenericTestData<>(10, "10")); assertNull( - genericMessageConverter.toMessage( - Collections.emptyList(), RqueueMessageHeaders.emptyMessageHeaders())); + genericMessageConverter.toMessage(items, RqueueMessageHeaders.emptyMessageHeaders())); } @Test @@ -220,65 +281,6 @@ public void testMultiLevelGenericTestDataFixedTypeToFromMessage() { assertEquals(data, fromMessage); } - @Data - @NoArgsConstructor - public static class MultiLevelGenericTestDataNoArgs { - private String data; - private GenericTestData tGenericTestData; - private GenericTestData vGenericTestData; - } - - @Data - @AllArgsConstructor - @NoArgsConstructor - public static class MultiLevelGenericTestData { - private String data; - private GenericTestData tGenericTestData; - private GenericTestData vGenericTestData; - } - - @Data - @AllArgsConstructor - @NoArgsConstructor - public static class MultiLevelGenericTestDataFixedType { - private String data; - private GenericTestData tGenericTestData; - private MultiGenericTestData vGenericTestData; - } - - @Data - @AllArgsConstructor - @NoArgsConstructor - public static class MultiGenericTestData { - private Integer index; - private K key; - private V value; - } - - @Data - @AllArgsConstructor - @NoArgsConstructor - public static class GenericTestData { - private Integer index; - private T data; - } - - @Data - @AllArgsConstructor - @NoArgsConstructor - public static class Comment { - private String id; - private String message; - } - - @Data - @AllArgsConstructor - @NoArgsConstructor - public static class Email { - private String id; - private String subject; - } - // https://stackoverflow.com/questions/64873444/generic-class-type-parameter-detail-at-runtime static class MappingRegistrar { From a75703c5e0905c8cced0d886ddefed4a17644ea3 Mon Sep 17 00:00:00 2001 From: Sonu Kumar Date: Sat, 5 Dec 2020 12:26:45 +0530 Subject: [PATCH 18/23] Build status link updated to circle ci --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f2644dd7..f2765d45 100644 --- a/README.md +++ b/README.md @@ -3,11 +3,11 @@

Rqueue: Redis Queue,Task Queue, Delayed Queue for Spring and Spring Boot

-[![Build Status](https://travis-ci.org/sonus21/rqueue.svg?branch=master)](https://travis-ci.org/sonus21/rqueue) +[![Build Status](https://circleci.com/gh/sonus21/rqueue/tree/master.svg?style=shield)](https://circleci.com/gh/sonus21/rqueue/tree/master) [![Coverage Status](https://coveralls.io/repos/github/sonus21/rqueue/badge.svg?branch=master)](https://coveralls.io/github/sonus21/rqueue?branch=master) [![Maven Central](https://img.shields.io/maven-central/v/com.github.sonus21/rqueue-core)](https://repo1.maven.org/maven2/com/github/sonus21/rqueue-core) -[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](LICENSE) [![Javadoc](https://javadoc.io/badge2/com.github.sonus21/rqueue-core/javadoc.svg)](https://javadoc.io/doc/com.github.sonus21/rqueue-core) +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](LICENSE) **Rqueue** is an asynchronous task executor(worker) built for spring framework based on the spring framework's messaging library backed by Redis. It can be used as message broker as well, where all services code is in Spring. From dda736017a6c94fe8db09b847c3fa1dcf8ffe2a2 Mon Sep 17 00:00:00 2001 From: Sonu Kumar Date: Sat, 5 Dec 2020 13:02:15 +0530 Subject: [PATCH 19/23] sleep time based on the waitTime --- .../sonus21/rqueue/utils/TimeoutUtils.java | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/TimeoutUtils.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/TimeoutUtils.java index a0aa32ab..8c1c329d 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/TimeoutUtils.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/TimeoutUtils.java @@ -48,6 +48,22 @@ public static void waitFor(BooleanSupplier callback, String description, Runnabl waitFor(callback, 10000L, description, postmortem); } + private static long getSleepTime(long waitTime) { + if (waitTime < 1000L) { + return 25L; + } + if (waitTime < 5_000L) { + return 50L; + } + if (waitTime < 10_000L) { + return 100L; + } + if (waitTime < 20_000) { + return 200L; + } + return waitTime / 60; + } + public static void waitFor( BooleanSupplier callback, long waitTimeInMilliSeconds, @@ -55,10 +71,7 @@ public static void waitFor( Runnable postmortem) throws TimedOutException { long endTime = System.currentTimeMillis() + waitTimeInMilliSeconds; - long sleepTime = 100L; - if (waitTimeInMilliSeconds > 2000) { - sleepTime = 200; - } + long sleepTime = getSleepTime(waitTimeInMilliSeconds); do { if (Boolean.TRUE.equals(callback.getAsBoolean())) { return; From cf37ee61d8fe1b28a1963622085dc813c9b3e5c5 Mon Sep 17 00:00:00 2001 From: Sonu Kumar Date: Sat, 5 Dec 2020 13:37:38 +0530 Subject: [PATCH 20/23] Allow monitor via config --- .../ApplicationBasicConfiguration.java | 12 ++++++++++++ .../rqueue/converter/RqueueRedisSerializer.java | 16 ++++++++++++++++ .../sonus21/rqueue/listener/RqueueExecutor.java | 6 +++++- .../main/resources/scripts/schedule_message.lua | 4 +--- .../tests/integration/PeriodicMessageTest.java | 1 + 5 files changed, 35 insertions(+), 4 deletions(-) diff --git a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/application/ApplicationBasicConfiguration.java b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/application/ApplicationBasicConfiguration.java index a3ec94fd..b79eaa1f 100644 --- a/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/application/ApplicationBasicConfiguration.java +++ b/rqueue-common-test/src/main/java/com/github/sonus21/rqueue/test/application/ApplicationBasicConfiguration.java @@ -26,6 +26,7 @@ import java.util.concurrent.Executors; import javax.sql.DataSource; import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; @@ -37,6 +38,7 @@ import org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter; import redis.embedded.RedisServer; +@Slf4j public abstract class ApplicationBasicConfiguration { private static final Logger monitorLogger = LoggerFactory.getLogger("monitor"); protected RedisServer redisServer; @@ -58,11 +60,20 @@ public abstract class ApplicationBasicConfiguration { @Value("${monitor.thread.count:0}") protected int monitorThreads; + @Value("${monitor.enabled:false}") + protected boolean monitoringEnabled; + protected void init() { + if (monitoringEnabled && monitorThreads == 0) { + monitorThreads = 1; + } if (monitorThreads > 0) { executorService = Executors.newFixedThreadPool(monitorThreads); processes = new ArrayList<>(); } + if (monitoringEnabled) { + monitor(redisHost, redisPort); + } if (useSystemRedis) { return; } @@ -92,6 +103,7 @@ protected void destroy() { } protected void monitor(String host, int port) { + log.info("Monitor {}:{}", host, port); executorService.submit( () -> { try { diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/RqueueRedisSerializer.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/RqueueRedisSerializer.java index 509cbbb9..9a0b91dc 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/RqueueRedisSerializer.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/converter/RqueueRedisSerializer.java @@ -1,3 +1,19 @@ +/* + * Copyright 2020 Sonu Kumar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.github.sonus21.rqueue.converter; import com.github.sonus21.rqueue.utils.SerializationUtils; diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueExecutor.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueExecutor.java index 1e516640..b39d1a34 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueExecutor.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/listener/RqueueExecutor.java @@ -236,7 +236,11 @@ private void processPeriodicMessage() { // avoid duplicate message enqueue due to retry by checking the message key // avoid cross slot error by using tagged queue name in the key String messageId = - queueDetail.getQueueName() + rqueueMessage.getId() + "::sch::" + newMessage.getProcessAt(); + queueDetail.getQueueName() + + "::" + + rqueueMessage.getId() + + "::sch::" + + newMessage.getProcessAt(); log.debug( "Schedule periodic message: {} Status: {}", rqueueMessage, diff --git a/rqueue-core/src/main/resources/scripts/schedule_message.lua b/rqueue-core/src/main/resources/scripts/schedule_message.lua index 097e3faf..7d89fdfa 100644 --- a/rqueue-core/src/main/resources/scripts/schedule_message.lua +++ b/rqueue-core/src/main/resources/scripts/schedule_message.lua @@ -1,11 +1,9 @@ -- get current value local value = redis.call('GET', KEYS[1]) - if value then return 0 end - -redis.call('SET', KEYS[1], "1", "EX", ARGV[1]) +redis.call('SET', KEYS[1], '1', 'EX', ARGV[1]) redis.call('ZADD', KEYS[2], ARGV[3], ARGV[2]) return 1 diff --git a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/PeriodicMessageTest.java b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/PeriodicMessageTest.java index c6c583d1..182a557e 100644 --- a/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/PeriodicMessageTest.java +++ b/rqueue-spring-boot-starter/src/test/java/com/github/sonus21/rqueue/spring/boot/tests/integration/PeriodicMessageTest.java @@ -43,6 +43,7 @@ "rqueue.metrics.count.execution=false", "periodic.job.queue.active=true", "use.system.redis=false", + "monitor.enabled=false" }) public class PeriodicMessageTest extends SpringTestBase { @Test From 38866d4094ea13f61f10d490a11e41ee1930d4d4 Mon Sep 17 00:00:00 2001 From: Sonu Kumar Date: Sat, 5 Dec 2020 13:43:00 +0530 Subject: [PATCH 21/23] Fixed sleep time --- .../sonus21/rqueue/utils/TimeoutUtils.java | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/TimeoutUtils.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/TimeoutUtils.java index 8c1c329d..26157e57 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/TimeoutUtils.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/utils/TimeoutUtils.java @@ -48,22 +48,6 @@ public static void waitFor(BooleanSupplier callback, String description, Runnabl waitFor(callback, 10000L, description, postmortem); } - private static long getSleepTime(long waitTime) { - if (waitTime < 1000L) { - return 25L; - } - if (waitTime < 5_000L) { - return 50L; - } - if (waitTime < 10_000L) { - return 100L; - } - if (waitTime < 20_000) { - return 200L; - } - return waitTime / 60; - } - public static void waitFor( BooleanSupplier callback, long waitTimeInMilliSeconds, @@ -71,7 +55,7 @@ public static void waitFor( Runnable postmortem) throws TimedOutException { long endTime = System.currentTimeMillis() + waitTimeInMilliSeconds; - long sleepTime = getSleepTime(waitTimeInMilliSeconds); + long sleepTime = 100; do { if (Boolean.TRUE.equals(callback.getAsBoolean())) { return; From d97494291bb5feb42fb904348f9b760a44be7bd0 Mon Sep 17 00:00:00 2001 From: Sonu Kumar Date: Sat, 5 Dec 2020 17:25:21 +0530 Subject: [PATCH 22/23] execution time delta should be only done for non-periodic task. --- .../rqueue/core/impl/RqueueMessageTemplateImpl.java | 2 +- .../github/sonus21/rqueue/models/db/MessageMetadata.java | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageTemplateImpl.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageTemplateImpl.java index 56817288..e636b4b5 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageTemplateImpl.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/core/impl/RqueueMessageTemplateImpl.java @@ -45,7 +45,7 @@ @Slf4j public class RqueueMessageTemplateImpl extends RqueueRedisTemplate implements RqueueMessageTemplate { - private DefaultScriptExecutor scriptExecutor; + private final DefaultScriptExecutor scriptExecutor; public RqueueMessageTemplateImpl(RedisConnectionFactory redisConnectionFactory) { super(redisConnectionFactory); diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/db/MessageMetadata.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/db/MessageMetadata.java index 450c6283..54b2ce24 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/db/MessageMetadata.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/db/MessageMetadata.java @@ -52,6 +52,12 @@ public MessageMetadata(RqueueMessage rqueueMessage, MessageStatus messageStatus) } public void addExecutionTime(long jobStartTime) { - this.totalExecutionTime += (System.currentTimeMillis() - jobStartTime); + long executionTime = (System.currentTimeMillis() - jobStartTime); + if (totalExecutionTime > 0 && !rqueueMessage.isPeriodicTask()) { + this.totalExecutionTime += executionTime; + } else { + // for non periodic job don't add execution time as the same job id would be running + this.totalExecutionTime = executionTime; + } } } From 65289b7b91541425612307b6534da00e402b272f Mon Sep 17 00:00:00 2001 From: Sonu Kumar Date: Sat, 5 Dec 2020 18:07:10 +0530 Subject: [PATCH 23/23] check for rqueueMessage being null --- .../com/github/sonus21/rqueue/models/db/MessageMetadata.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/db/MessageMetadata.java b/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/db/MessageMetadata.java index 54b2ce24..0c1886c8 100644 --- a/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/db/MessageMetadata.java +++ b/rqueue-core/src/main/java/com/github/sonus21/rqueue/models/db/MessageMetadata.java @@ -53,7 +53,7 @@ public MessageMetadata(RqueueMessage rqueueMessage, MessageStatus messageStatus) public void addExecutionTime(long jobStartTime) { long executionTime = (System.currentTimeMillis() - jobStartTime); - if (totalExecutionTime > 0 && !rqueueMessage.isPeriodicTask()) { + if (rqueueMessage != null && totalExecutionTime > 0 && !rqueueMessage.isPeriodicTask()) { this.totalExecutionTime += executionTime; } else { // for non periodic job don't add execution time as the same job id would be running