diff --git a/build.gradle b/build.gradle index 84b850df..c31cfc4c 100644 --- a/build.gradle +++ b/build.gradle @@ -39,9 +39,11 @@ dependencies { implementation 'com.google.cloud:google-cloud-bigquery:1.115.0' implementation "io.grpc:grpc-all:1.38.0" implementation group: 'org.slf4j', name: 'jul-to-slf4j', version: '1.7.35' + implementation group: 'redis.clients', name: 'jedis', version: '3.0.1' + implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.5' implementation 'com.fasterxml.jackson.core:jackson-databind:2.13.2.1' implementation 'org.json:json:20220320' - + implementation group: 'com.jayway.jsonpath', name: 'json-path', version: '2.4.0' testImplementation group: 'junit', name: 'junit', version: '4.13' testImplementation 'org.hamcrest:hamcrest-all:1.3' testImplementation 'org.mockito:mockito-core:4.5.1' diff --git a/src/main/java/io/odpf/depot/config/OdpfSinkConfig.java b/src/main/java/io/odpf/depot/config/OdpfSinkConfig.java index cb89cd7d..277151e1 100644 --- a/src/main/java/io/odpf/depot/config/OdpfSinkConfig.java +++ b/src/main/java/io/odpf/depot/config/OdpfSinkConfig.java @@ -64,6 +64,10 @@ public interface OdpfSinkConfig extends Config { @DefaultValue("") String getSinkConnectorSchemaProtoKeyClass(); + @Key("SINK_CONNECTOR_SCHEMA_JSON_PARSER_STRING_MODE_ENABLED") + @DefaultValue("true") + boolean getSinkConnectorSchemaJsonParserStringModeEnabled(); + @Key("SINK_CONNECTOR_SCHEMA_DATA_TYPE") @ConverterClass(SinkConnectorSchemaDataTypeConverter.class) @DefaultValue("PROTOBUF") diff --git a/src/main/java/io/odpf/depot/config/RedisSinkConfig.java b/src/main/java/io/odpf/depot/config/RedisSinkConfig.java new file mode 100644 index 00000000..cdd4731c --- /dev/null +++ b/src/main/java/io/odpf/depot/config/RedisSinkConfig.java @@ -0,0 +1,55 @@ +package io.odpf.depot.config; + +import io.odpf.depot.config.converter.JsonToPropertiesConverter; +import io.odpf.depot.config.converter.RedisSinkDataTypeConverter; +import io.odpf.depot.config.converter.RedisSinkDeploymentTypeConverter; +import io.odpf.depot.config.converter.RedisSinkTtlTypeConverter; +import io.odpf.depot.redis.enums.RedisSinkDataType; +import io.odpf.depot.redis.enums.RedisSinkDeploymentType; +import io.odpf.depot.redis.enums.RedisSinkTtlType; +import org.aeonbits.owner.Config; + +import java.util.Properties; + + +@Config.DisableFeature(Config.DisableableFeature.PARAMETER_FORMATTING) +public interface RedisSinkConfig extends OdpfSinkConfig { + @Key("SINK_REDIS_URLS") + String getSinkRedisUrls(); + + @Key("SINK_REDIS_KEY_TEMPLATE") + String getSinkRedisKeyTemplate(); + + @Key("SINK_REDIS_DATA_TYPE") + @DefaultValue("HASHSET") + @ConverterClass(RedisSinkDataTypeConverter.class) + RedisSinkDataType getSinkRedisDataType(); + + @Key("SINK_REDIS_TTL_TYPE") + @DefaultValue("DISABLE") + @ConverterClass(RedisSinkTtlTypeConverter.class) + RedisSinkTtlType getSinkRedisTtlType(); + + @Key("SINK_REDIS_TTL_VALUE") + @DefaultValue("0") + long getSinkRedisTtlValue(); + + @Key("SINK_REDIS_DEPLOYMENT_TYPE") + @DefaultValue("Standalone") + @ConverterClass(RedisSinkDeploymentTypeConverter.class) + RedisSinkDeploymentType getSinkRedisDeploymentType(); + + @Key("SINK_REDIS_LIST_DATA_PROTO_INDEX") + String getSinkRedisListDataProtoIndex(); + + @Key("SINK_REDIS_KEY_VALUE_DATA_FIELD_NAME") + String getSinkRedisKeyValueDataFieldName(); + + @Key("SINK_REDIS_LIST_DATA_FIELD_NAME") + String getSinkRedisListDataFieldName(); + + @Key("SINK_REDIS_HASHSET_FIELD_TO_COLUMN_MAPPING") + @ConverterClass(JsonToPropertiesConverter.class) + @DefaultValue("") + Properties getSinkRedisHashsetFieldToColumnMapping(); +} diff --git a/src/main/java/io/odpf/depot/config/converter/JsonToPropertiesConverter.java b/src/main/java/io/odpf/depot/config/converter/JsonToPropertiesConverter.java new file mode 100644 index 00000000..8c7d26ad --- /dev/null +++ b/src/main/java/io/odpf/depot/config/converter/JsonToPropertiesConverter.java @@ -0,0 +1,95 @@ +package io.odpf.depot.config.converter; + +import com.google.common.base.Strings; +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; + +import java.lang.reflect.Method; +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.function.Consumer; +import java.util.stream.Stream; + + +public class JsonToPropertiesConverter implements org.aeonbits.owner.Converter { + private static final Gson GSON = new Gson(); + + @Override + public Properties convert(Method method, String input) { + if (Strings.isNullOrEmpty(input)) { + return null; + } + Type type = new TypeToken>() { + }.getType(); + Map m = GSON.fromJson(input, type); + Properties properties = getProperties(m); + validate(properties); + return properties; + } + + private Properties getProperties(Map inputMap) { + Properties properties = new Properties(); + for (String key : inputMap.keySet()) { + Object value = inputMap.get(key); + if (value instanceof String) { + properties.put(key, value); + } else if (value instanceof Map) { + properties.put(key, getProperties((Map) value)); + } + } + return properties; + } + + private void validate(Properties properties) { + DuplicateFinder duplicateFinder = flattenValues(properties) + .collect(DuplicateFinder::new, DuplicateFinder::accept, DuplicateFinder::combine); + if (duplicateFinder.duplicates.size() > 0) { + throw new IllegalArgumentException("duplicates found in SINK_REDIS_HASHSET_FIELD_TO_COLUMN_MAPPING for : " + duplicateFinder.duplicates); + } + } + + private Stream flattenValues(Properties properties) { + return properties + .values() + .stream() + .flatMap(v -> { + if (v instanceof String) { + return Stream.of((String) v); + } else if (v instanceof Properties) { + return flattenValues((Properties) v); + } else { + return Stream.empty(); + } + }); + } + + private static class DuplicateFinder implements Consumer { + private final Set processedValues = new HashSet<>(); + private final List duplicates = new ArrayList<>(); + + @Override + public void accept(String o) { + if (processedValues.contains(o)) { + duplicates.add(o); + } else { + processedValues.add(o); + } + } + + void combine(DuplicateFinder other) { + other.processedValues + .forEach(v -> { + if (processedValues.contains(v)) { + duplicates.add(v); + } else { + processedValues.add(v); + } + }); + } + } +} diff --git a/src/main/java/io/odpf/depot/config/converter/RedisSinkDataTypeConverter.java b/src/main/java/io/odpf/depot/config/converter/RedisSinkDataTypeConverter.java new file mode 100644 index 00000000..b16bdd89 --- /dev/null +++ b/src/main/java/io/odpf/depot/config/converter/RedisSinkDataTypeConverter.java @@ -0,0 +1,13 @@ +package io.odpf.depot.config.converter; + +import io.odpf.depot.redis.enums.RedisSinkDataType; +import org.aeonbits.owner.Converter; + +import java.lang.reflect.Method; + +public class RedisSinkDataTypeConverter implements Converter { + @Override + public RedisSinkDataType convert(Method method, String input) { + return RedisSinkDataType.valueOf(input.toUpperCase()); + } +} diff --git a/src/main/java/io/odpf/depot/config/converter/RedisSinkDeploymentTypeConverter.java b/src/main/java/io/odpf/depot/config/converter/RedisSinkDeploymentTypeConverter.java new file mode 100644 index 00000000..d8dfff02 --- /dev/null +++ b/src/main/java/io/odpf/depot/config/converter/RedisSinkDeploymentTypeConverter.java @@ -0,0 +1,13 @@ +package io.odpf.depot.config.converter; + +import io.odpf.depot.redis.enums.RedisSinkDeploymentType; +import org.aeonbits.owner.Converter; + +import java.lang.reflect.Method; + +public class RedisSinkDeploymentTypeConverter implements Converter { + @Override + public RedisSinkDeploymentType convert(Method method, String input) { + return RedisSinkDeploymentType.valueOf(input.toUpperCase()); + } +} diff --git a/src/main/java/io/odpf/depot/config/converter/RedisSinkTtlTypeConverter.java b/src/main/java/io/odpf/depot/config/converter/RedisSinkTtlTypeConverter.java new file mode 100644 index 00000000..88071043 --- /dev/null +++ b/src/main/java/io/odpf/depot/config/converter/RedisSinkTtlTypeConverter.java @@ -0,0 +1,13 @@ +package io.odpf.depot.config.converter; + +import io.odpf.depot.redis.enums.RedisSinkTtlType; +import org.aeonbits.owner.Converter; + +import java.lang.reflect.Method; + +public class RedisSinkTtlTypeConverter implements Converter { + @Override + public RedisSinkTtlType convert(Method method, String input) { + return RedisSinkTtlType.valueOf(input.toUpperCase()); + } +} diff --git a/src/main/java/io/odpf/depot/error/ErrorInfo.java b/src/main/java/io/odpf/depot/error/ErrorInfo.java index 33346078..7d899d3c 100644 --- a/src/main/java/io/odpf/depot/error/ErrorInfo.java +++ b/src/main/java/io/odpf/depot/error/ErrorInfo.java @@ -8,10 +8,11 @@ @Data public class ErrorInfo { - @EqualsAndHashCode.Exclude private Exception exception; + @EqualsAndHashCode.Exclude + private Exception exception; private ErrorType errorType; public String toString() { - return errorType.name(); + return String.format("Exception %s, ErrorType: %s", exception != null ? exception.getMessage() : "NULL", errorType.name()); } } diff --git a/src/main/java/io/odpf/depot/message/OdpfMessageSchema.java b/src/main/java/io/odpf/depot/message/OdpfMessageSchema.java index 3f6ab003..62c09938 100644 --- a/src/main/java/io/odpf/depot/message/OdpfMessageSchema.java +++ b/src/main/java/io/odpf/depot/message/OdpfMessageSchema.java @@ -1,8 +1,5 @@ package io.odpf.depot.message; - -import java.io.IOException; - public interface OdpfMessageSchema { - Object getSchema() throws IOException; + Object getSchema(); } diff --git a/src/main/java/io/odpf/depot/message/ParsedOdpfMessage.java b/src/main/java/io/odpf/depot/message/ParsedOdpfMessage.java index cdc2a7a5..8f96fa69 100644 --- a/src/main/java/io/odpf/depot/message/ParsedOdpfMessage.java +++ b/src/main/java/io/odpf/depot/message/ParsedOdpfMessage.java @@ -11,4 +11,6 @@ public interface ParsedOdpfMessage { void validate(OdpfSinkConfig config); Map getMapping(OdpfMessageSchema schema) throws IOException; + + Object getFieldByName(String name, OdpfMessageSchema odpfMessageSchema); } diff --git a/src/main/java/io/odpf/depot/message/json/JsonOdpfMessageParser.java b/src/main/java/io/odpf/depot/message/json/JsonOdpfMessageParser.java index 6c86831f..0fa8c694 100644 --- a/src/main/java/io/odpf/depot/message/json/JsonOdpfMessageParser.java +++ b/src/main/java/io/odpf/depot/message/json/JsonOdpfMessageParser.java @@ -10,6 +10,7 @@ import io.odpf.depot.message.ParsedOdpfMessage; import io.odpf.depot.metrics.Instrumentation; import io.odpf.depot.metrics.JsonParserMetrics; +import io.odpf.depot.utils.JsonUtils; import lombok.extern.slf4j.Slf4j; import org.json.JSONException; import org.json.JSONObject; @@ -54,21 +55,9 @@ public ParsedOdpfMessage parse(OdpfMessage message, SinkConnectorSchemaMessageMo throw new EmptyMessageException(); } Instant instant = Instant.now(); - JSONObject jsonObject = new JSONObject(new String(payload)); - JSONObject jsonWithStringValues = new JSONObject(); - jsonObject.keySet() - .forEach(k -> { - Object value = jsonObject.get(k); - if (value instanceof JSONObject) { - throw new UnsupportedOperationException("nested json structure not supported yet"); - } - if (JSONObject.NULL.equals(value)) { - return; - } - jsonWithStringValues.put(k, value.toString()); - }); + JSONObject jsonObject = JsonUtils.getJsonObject(config, payload); instrumentation.captureDurationSince(jsonParserMetrics.getJsonParseTimeTakenMetric(), instant); - return new JsonOdpfParsedMessage(jsonWithStringValues); + return new JsonOdpfParsedMessage(jsonObject); } catch (JSONException ex) { throw new IOException("invalid json error", ex); } diff --git a/src/main/java/io/odpf/depot/message/json/JsonOdpfParsedMessage.java b/src/main/java/io/odpf/depot/message/json/JsonOdpfParsedMessage.java index 69c73cd2..5ee29ab4 100644 --- a/src/main/java/io/odpf/depot/message/json/JsonOdpfParsedMessage.java +++ b/src/main/java/io/odpf/depot/message/json/JsonOdpfParsedMessage.java @@ -1,5 +1,8 @@ package io.odpf.depot.message.json; +import com.jayway.jsonpath.Configuration; +import com.jayway.jsonpath.JsonPath; +import com.jayway.jsonpath.spi.json.JsonOrgJsonProvider; import io.odpf.depot.config.OdpfSinkConfig; import io.odpf.depot.message.OdpfMessageSchema; import io.odpf.depot.message.ParsedOdpfMessage; @@ -32,8 +35,17 @@ public void validate(OdpfSinkConfig config) { @Override public Map getMapping(OdpfMessageSchema schema) { if (jsonObject == null || jsonObject.isEmpty()) { - return Collections.emptyMap(); + return Collections.emptyMap(); } return jsonObject.toMap(); } + + public Object getFieldByName(String name, OdpfMessageSchema odpfMessageSchema) { + String jsonPathName = "$." + name; + Configuration configuration = Configuration.builder() + .jsonProvider(new JsonOrgJsonProvider()) + .build(); + JsonPath jsonPath = JsonPath.compile(jsonPathName); + return jsonPath.read(jsonObject, configuration); + } } diff --git a/src/main/java/io/odpf/depot/message/proto/ProtoField.java b/src/main/java/io/odpf/depot/message/proto/ProtoField.java index dc01af8f..965568cd 100644 --- a/src/main/java/io/odpf/depot/message/proto/ProtoField.java +++ b/src/main/java/io/odpf/depot/message/proto/ProtoField.java @@ -1,11 +1,13 @@ package io.odpf.depot.message.proto; import com.google.protobuf.DescriptorProtos; +import lombok.EqualsAndHashCode; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +@EqualsAndHashCode public class ProtoField { private String name; private String typeName; diff --git a/src/main/java/io/odpf/depot/message/proto/ProtoOdpfMessageSchema.java b/src/main/java/io/odpf/depot/message/proto/ProtoOdpfMessageSchema.java index 3023c466..e97ca805 100644 --- a/src/main/java/io/odpf/depot/message/proto/ProtoOdpfMessageSchema.java +++ b/src/main/java/io/odpf/depot/message/proto/ProtoOdpfMessageSchema.java @@ -3,6 +3,7 @@ import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; import io.odpf.depot.message.OdpfMessageSchema; +import lombok.EqualsAndHashCode; import lombok.Getter; import java.io.IOException; @@ -10,6 +11,7 @@ import java.util.Map; import java.util.Properties; +@EqualsAndHashCode public class ProtoOdpfMessageSchema implements OdpfMessageSchema { @Getter @@ -21,13 +23,13 @@ public ProtoOdpfMessageSchema(ProtoField protoField) throws IOException { this(protoField, createProperties(protoField)); } - public ProtoOdpfMessageSchema(ProtoField protoField, Properties properties) throws IOException { + public ProtoOdpfMessageSchema(ProtoField protoField, Properties properties) { this.protoField = protoField; this.properties = properties; } @Override - public Properties getSchema() throws IOException { + public Properties getSchema() { return this.properties; } diff --git a/src/main/java/io/odpf/depot/message/proto/ProtoOdpfParsedMessage.java b/src/main/java/io/odpf/depot/message/proto/ProtoOdpfParsedMessage.java index 23f911d0..886b5bec 100644 --- a/src/main/java/io/odpf/depot/message/proto/ProtoOdpfParsedMessage.java +++ b/src/main/java/io/odpf/depot/message/proto/ProtoOdpfParsedMessage.java @@ -14,7 +14,6 @@ import io.odpf.depot.utils.ProtoUtils; import lombok.extern.slf4j.Slf4j; -import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -25,6 +24,8 @@ public class ProtoOdpfParsedMessage implements ParsedOdpfMessage { private final DynamicMessage dynamicMessage; + private final Map> cachedMapping = new HashMap<>(); + public ProtoOdpfParsedMessage(DynamicMessage dynamicMessage) { this.dynamicMessage = dynamicMessage; } @@ -47,11 +48,11 @@ public void validate(OdpfSinkConfig config) { } @Override - public Map getMapping(OdpfMessageSchema schema) throws IOException { + public Map getMapping(OdpfMessageSchema schema) { if (schema.getSchema() == null) { - throw new ConfigurationException("BQ_PROTO_COLUMN_MAPPING is not configured"); + throw new ConfigurationException("Schema is not configured"); } - return getMappings(dynamicMessage, (Properties) schema.getSchema()); + return cachedMapping.computeIfAbsent(schema, x -> getMappings(dynamicMessage, (Properties) schema.getSchema())); } @SuppressWarnings("unchecked") @@ -125,4 +126,22 @@ private void addRepeatedFields(Map row, Object value, List fields = getMapping(odpfMessageSchema); + for (String key: keys) { + Object localValue = fields.get(key); + if (localValue == null) { + throw new ConfigurationException("Invalid field config : " + name); + } + if (localValue instanceof Map) { + fields = (Map) localValue; + } else { + return localValue; + } + } + return fields; + } } diff --git a/src/main/java/io/odpf/depot/redis/RedisSink.java b/src/main/java/io/odpf/depot/redis/RedisSink.java new file mode 100644 index 00000000..6407142a --- /dev/null +++ b/src/main/java/io/odpf/depot/redis/RedisSink.java @@ -0,0 +1,51 @@ +package io.odpf.depot.redis; + +import io.odpf.depot.OdpfSink; +import io.odpf.depot.OdpfSinkResponse; +import io.odpf.depot.error.ErrorInfo; +import io.odpf.depot.message.OdpfMessage; +import io.odpf.depot.metrics.Instrumentation; +import io.odpf.depot.redis.client.RedisClient; +import io.odpf.depot.redis.client.response.RedisResponse; +import io.odpf.depot.redis.parsers.RedisParser; +import io.odpf.depot.redis.util.RedisSinkUtils; +import io.odpf.depot.redis.record.RedisRecord; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class RedisSink implements OdpfSink { + private final RedisClient redisClient; + private final RedisParser redisParser; + private final Instrumentation instrumentation; + + public RedisSink(RedisClient redisClient, RedisParser redisParser, Instrumentation instrumentation) { + this.redisClient = redisClient; + this.redisParser = redisParser; + this.instrumentation = instrumentation; + } + + @Override + public OdpfSinkResponse pushToSink(List messages) { + List records = redisParser.convert(messages); + Map> splitterRecords = records.stream().collect(Collectors.partitioningBy(RedisRecord::isValid)); + List invalidRecords = splitterRecords.get(Boolean.FALSE); + List validRecords = splitterRecords.get(Boolean.TRUE); + OdpfSinkResponse odpfSinkResponse = new OdpfSinkResponse(); + invalidRecords.forEach(invalidRecord -> odpfSinkResponse.addErrors(invalidRecord.getIndex(), invalidRecord.getErrorInfo())); + if (validRecords.size() > 0) { + List responses = redisClient.send(validRecords); + Map errorInfoMap = RedisSinkUtils.getErrorsFromResponse(validRecords, responses, instrumentation); + errorInfoMap.forEach(odpfSinkResponse::addErrors); + instrumentation.logInfo("Pushed a batch of {} records to Redis", validRecords.size()); + } + return odpfSinkResponse; + } + + @Override + public void close() throws IOException { + + } +} diff --git a/src/main/java/io/odpf/depot/redis/RedisSinkFactory.java b/src/main/java/io/odpf/depot/redis/RedisSinkFactory.java new file mode 100644 index 00000000..d8132ebb --- /dev/null +++ b/src/main/java/io/odpf/depot/redis/RedisSinkFactory.java @@ -0,0 +1,68 @@ +package io.odpf.depot.redis; + + +import com.timgroup.statsd.NoOpStatsDClient; +import io.odpf.depot.OdpfSink; +import io.odpf.depot.common.Tuple; +import io.odpf.depot.config.RedisSinkConfig; +import io.odpf.depot.message.OdpfMessageParser; +import io.odpf.depot.message.OdpfMessageParserFactory; +import io.odpf.depot.message.OdpfMessageSchema; +import io.odpf.depot.message.SinkConnectorSchemaMessageMode; +import io.odpf.depot.metrics.Instrumentation; +import io.odpf.depot.metrics.StatsDReporter; +import io.odpf.depot.redis.client.RedisClientFactory; +import io.odpf.depot.redis.parsers.RedisEntryParser; +import io.odpf.depot.redis.parsers.RedisEntryParserFactory; +import io.odpf.depot.redis.parsers.RedisParser; +import io.odpf.depot.utils.MessageConfigUtils; + +import java.io.IOException; + +public class RedisSinkFactory { + private final RedisSinkConfig sinkConfig; + private final StatsDReporter statsDReporter; + private RedisParser redisParser; + + public RedisSinkFactory(RedisSinkConfig sinkConfig, StatsDReporter statsDReporter) { + this.sinkConfig = sinkConfig; + this.statsDReporter = statsDReporter; + } + + public RedisSinkFactory(RedisSinkConfig sinkConfig) { + this.sinkConfig = sinkConfig; + this.statsDReporter = new StatsDReporter(new NoOpStatsDClient()); + } + + public void init() throws IOException { + Instrumentation instrumentation = new Instrumentation(statsDReporter, RedisSinkFactory.class); + String redisConfig = String.format("\n\tredis.urls = %s\n\tredis.key.template = %s\n\tredis.sink.type = %s" + + "\n\tredis.list.data.proto.index = %s\n\tredis.ttl.type = %s\n\tredis.ttl.value = %d", + sinkConfig.getSinkRedisUrls(), + sinkConfig.getSinkRedisKeyTemplate(), + sinkConfig.getSinkRedisDataType().toString(), + sinkConfig.getSinkRedisListDataProtoIndex(), + sinkConfig.getSinkRedisTtlType().toString(), + sinkConfig.getSinkRedisTtlValue()); + instrumentation.logInfo(redisConfig); + instrumentation.logInfo("Redis server type = {}", sinkConfig.getSinkRedisDeploymentType()); + OdpfMessageParser messageParser = OdpfMessageParserFactory.getParser(sinkConfig, statsDReporter); + Tuple modeAndSchema = MessageConfigUtils.getModeAndSchema(sinkConfig); + OdpfMessageSchema schema = messageParser.getSchema(modeAndSchema.getSecond()); + RedisEntryParser redisEntryParser = RedisEntryParserFactory.getRedisEntryParser(sinkConfig, statsDReporter, schema); + this.redisParser = new RedisParser(messageParser, redisEntryParser, modeAndSchema); + instrumentation.logInfo("Connection to redis established successfully"); + } + + /** + * We create redis client for each create call, because it's not thread safe. + * + * @return RedisSink + */ + public OdpfSink create() { + return new RedisSink( + RedisClientFactory.getClient(sinkConfig, statsDReporter), + redisParser, + new Instrumentation(statsDReporter, RedisSink.class)); + } +} diff --git a/src/main/java/io/odpf/depot/redis/client/RedisClient.java b/src/main/java/io/odpf/depot/redis/client/RedisClient.java new file mode 100644 index 00000000..16d4894b --- /dev/null +++ b/src/main/java/io/odpf/depot/redis/client/RedisClient.java @@ -0,0 +1,14 @@ +package io.odpf.depot.redis.client; + +import io.odpf.depot.redis.client.response.RedisResponse; +import io.odpf.depot.redis.record.RedisRecord; + +import java.io.Closeable; +import java.util.List; + +/** + * Redis client interface to be used in RedisSink. + */ +public interface RedisClient extends Closeable { + List send(List records); +} diff --git a/src/main/java/io/odpf/depot/redis/client/RedisClientFactory.java b/src/main/java/io/odpf/depot/redis/client/RedisClientFactory.java new file mode 100644 index 00000000..57bc3995 --- /dev/null +++ b/src/main/java/io/odpf/depot/redis/client/RedisClientFactory.java @@ -0,0 +1,57 @@ +package io.odpf.depot.redis.client; + + +import io.odpf.depot.config.RedisSinkConfig; +import io.odpf.depot.exception.ConfigurationException; +import io.odpf.depot.metrics.Instrumentation; +import io.odpf.depot.metrics.StatsDReporter; +import io.odpf.depot.redis.enums.RedisSinkDeploymentType; +import io.odpf.depot.redis.ttl.RedisTTLFactory; +import io.odpf.depot.redis.ttl.RedisTtl; +import org.apache.commons.lang3.StringUtils; +import redis.clients.jedis.HostAndPort; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisCluster; + +import java.util.HashSet; + +/** + * Redis client factory. + */ +public class RedisClientFactory { + + private static final String DELIMITER = ","; + + public static RedisClient getClient(RedisSinkConfig redisSinkConfig, StatsDReporter statsDReporter) { + RedisSinkDeploymentType redisSinkDeploymentType = redisSinkConfig.getSinkRedisDeploymentType(); + RedisTtl redisTTL = RedisTTLFactory.getTTl(redisSinkConfig); + return RedisSinkDeploymentType.CLUSTER.equals(redisSinkDeploymentType) + ? getRedisClusterClient(redisTTL, redisSinkConfig, statsDReporter) + : getRedisStandaloneClient(redisTTL, redisSinkConfig, statsDReporter); + } + + private static RedisStandaloneClient getRedisStandaloneClient(RedisTtl redisTTL, RedisSinkConfig redisSinkConfig, StatsDReporter statsDReporter) { + HostAndPort hostAndPort; + try { + hostAndPort = HostAndPort.parseString(StringUtils.trim(redisSinkConfig.getSinkRedisUrls())); + } catch (IllegalArgumentException e) { + throw new ConfigurationException(String.format("Invalid url for redis standalone: %s", redisSinkConfig.getSinkRedisUrls())); + } + Jedis jedis = new Jedis(hostAndPort); + return new RedisStandaloneClient(new Instrumentation(statsDReporter, RedisStandaloneClient.class), redisTTL, jedis); + } + + private static RedisClusterClient getRedisClusterClient(RedisTtl redisTTL, RedisSinkConfig redisSinkConfig, StatsDReporter statsDReporter) { + String[] redisUrls = redisSinkConfig.getSinkRedisUrls().split(DELIMITER); + HashSet nodes = new HashSet<>(); + try { + for (String redisUrl : redisUrls) { + nodes.add(HostAndPort.parseString(StringUtils.trim(redisUrl))); + } + } catch (IllegalArgumentException e) { + throw new ConfigurationException(String.format("Invalid url(s) for redis cluster: %s", redisSinkConfig.getSinkRedisUrls())); + } + JedisCluster jedisCluster = new JedisCluster(nodes); + return new RedisClusterClient(new Instrumentation(statsDReporter, RedisClusterClient.class), redisTTL, jedisCluster); + } +} diff --git a/src/main/java/io/odpf/depot/redis/client/RedisClusterClient.java b/src/main/java/io/odpf/depot/redis/client/RedisClusterClient.java new file mode 100644 index 00000000..7dea53fe --- /dev/null +++ b/src/main/java/io/odpf/depot/redis/client/RedisClusterClient.java @@ -0,0 +1,35 @@ +package io.odpf.depot.redis.client; + +import io.odpf.depot.metrics.Instrumentation; +import io.odpf.depot.redis.client.response.RedisResponse; +import io.odpf.depot.redis.record.RedisRecord; +import io.odpf.depot.redis.ttl.RedisTtl; +import lombok.AllArgsConstructor; +import redis.clients.jedis.JedisCluster; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * Redis cluster client. + */ +@AllArgsConstructor +public class RedisClusterClient implements RedisClient { + + private final Instrumentation instrumentation; + private final RedisTtl redisTTL; + private final JedisCluster jedisCluster; + + @Override + public List send(List records) { + return records.stream() + .map(record -> record.send(jedisCluster, redisTTL)) + .collect(Collectors.toList()); + } + + @Override + public void close() { + instrumentation.logInfo("Closing Jedis client"); + jedisCluster.close(); + } +} diff --git a/src/main/java/io/odpf/depot/redis/client/RedisStandaloneClient.java b/src/main/java/io/odpf/depot/redis/client/RedisStandaloneClient.java new file mode 100644 index 00000000..12046cab --- /dev/null +++ b/src/main/java/io/odpf/depot/redis/client/RedisStandaloneClient.java @@ -0,0 +1,51 @@ +package io.odpf.depot.redis.client; + +import io.odpf.depot.metrics.Instrumentation; +import io.odpf.depot.redis.client.response.RedisResponse; +import io.odpf.depot.redis.client.response.RedisStandaloneResponse; +import io.odpf.depot.redis.record.RedisRecord; +import io.odpf.depot.redis.ttl.RedisTtl; +import lombok.AllArgsConstructor; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.Pipeline; +import redis.clients.jedis.Response; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * Redis standalone client. + */ +@AllArgsConstructor +public class RedisStandaloneClient implements RedisClient { + + private final Instrumentation instrumentation; + private final RedisTtl redisTTL; + private final Jedis jedis; + + /** + * Pushes records in a transaction. + * if the transaction fails, whole batch can be retried. + * + * @param records records to send + * @return Custom response containing status of the API calls. + */ + @Override + public List send(List records) { + Pipeline jedisPipelined = jedis.pipelined(); + jedisPipelined.multi(); + List responses = records.stream() + .map(redisRecord -> redisRecord.send(jedisPipelined, redisTTL)) + .collect(Collectors.toList()); + Response> executeResponse = jedisPipelined.exec(); + jedisPipelined.sync(); + instrumentation.logDebug("jedis responses: {}", executeResponse.get()); + return responses.stream().map(RedisStandaloneResponse::process).collect(Collectors.toList()); + } + + @Override + public void close() { + instrumentation.logInfo("Closing Jedis client"); + jedis.close(); + } +} diff --git a/src/main/java/io/odpf/depot/redis/client/entry/RedisEntry.java b/src/main/java/io/odpf/depot/redis/client/entry/RedisEntry.java new file mode 100644 index 00000000..2ced72ab --- /dev/null +++ b/src/main/java/io/odpf/depot/redis/client/entry/RedisEntry.java @@ -0,0 +1,29 @@ +package io.odpf.depot.redis.client.entry; + +import io.odpf.depot.redis.client.response.RedisClusterResponse; +import io.odpf.depot.redis.client.response.RedisStandaloneResponse; +import io.odpf.depot.redis.ttl.RedisTtl; +import redis.clients.jedis.JedisCluster; +import redis.clients.jedis.Pipeline; + +/** + * The interface Redis data entry. + */ +public interface RedisEntry { + + /** + * Push messages to jedis pipeline. + * + * @param jedisPipelined the jedis pipelined + * @param redisTTL the redis ttl + */ + RedisStandaloneResponse send(Pipeline jedisPipelined, RedisTtl redisTTL); + + /** + * Push message to jedis cluster. + * + * @param jedisCluster the jedis cluster + * @param redisTTL the redis ttl + */ + RedisClusterResponse send(JedisCluster jedisCluster, RedisTtl redisTTL); +} diff --git a/src/main/java/io/odpf/depot/redis/client/entry/RedisHashSetFieldEntry.java b/src/main/java/io/odpf/depot/redis/client/entry/RedisHashSetFieldEntry.java new file mode 100644 index 00000000..2bced4f7 --- /dev/null +++ b/src/main/java/io/odpf/depot/redis/client/entry/RedisHashSetFieldEntry.java @@ -0,0 +1,51 @@ +package io.odpf.depot.redis.client.entry; + +import io.odpf.depot.metrics.Instrumentation; +import io.odpf.depot.redis.client.response.RedisClusterResponse; +import io.odpf.depot.redis.client.response.RedisStandaloneResponse; +import io.odpf.depot.redis.ttl.RedisTtl; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import redis.clients.jedis.JedisCluster; +import redis.clients.jedis.Pipeline; +import redis.clients.jedis.Response; +import redis.clients.jedis.exceptions.JedisException; + +/** + * Class for Redis Hash set entry. + */ +@AllArgsConstructor +@EqualsAndHashCode +public class RedisHashSetFieldEntry implements RedisEntry { + + private final String key; + private final String field; + private final String value; + @EqualsAndHashCode.Exclude + private final Instrumentation instrumentation; + + @Override + public RedisStandaloneResponse send(Pipeline jedisPipelined, RedisTtl redisTTL) { + instrumentation.logDebug("key: {}, field: {}, value: {}", key, field, value); + Response response = jedisPipelined.hset(key, field, value); + Response ttlResponse = redisTTL.setTtl(jedisPipelined, key); + return new RedisStandaloneResponse("HSET", response, ttlResponse); + } + + @Override + public RedisClusterResponse send(JedisCluster jedisCluster, RedisTtl redisTTL) { + instrumentation.logDebug("key: {}, field: {}, value: {}", key, field, value); + try { + Long response = jedisCluster.hset(key, field, value); + Long ttlResponse = redisTTL.setTtl(jedisCluster, key); + return new RedisClusterResponse("HSET", response, ttlResponse); + } catch (JedisException e) { + return new RedisClusterResponse(e.getMessage()); + } + } + + @Override + public String toString() { + return String.format("RedisHashSetFieldEntry Key %s, Field %s, Value %s", key, field, value); + } +} diff --git a/src/main/java/io/odpf/depot/redis/client/entry/RedisKeyValueEntry.java b/src/main/java/io/odpf/depot/redis/client/entry/RedisKeyValueEntry.java new file mode 100644 index 00000000..7d55ffb4 --- /dev/null +++ b/src/main/java/io/odpf/depot/redis/client/entry/RedisKeyValueEntry.java @@ -0,0 +1,46 @@ +package io.odpf.depot.redis.client.entry; + +import io.odpf.depot.metrics.Instrumentation; +import io.odpf.depot.redis.client.response.RedisClusterResponse; +import io.odpf.depot.redis.client.response.RedisStandaloneResponse; +import io.odpf.depot.redis.ttl.RedisTtl; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import redis.clients.jedis.JedisCluster; +import redis.clients.jedis.Pipeline; +import redis.clients.jedis.Response; +import redis.clients.jedis.exceptions.JedisException; + +@AllArgsConstructor +@EqualsAndHashCode +public class RedisKeyValueEntry implements RedisEntry { + private final String key; + private final String value; + @EqualsAndHashCode.Exclude + private final Instrumentation instrumentation; + + @Override + public RedisStandaloneResponse send(Pipeline jedisPipelined, RedisTtl redisTTL) { + instrumentation.logDebug("key: {}, value: {}", key, value); + Response response = jedisPipelined.set(key, value); + Response ttlResponse = redisTTL.setTtl(jedisPipelined, key); + return new RedisStandaloneResponse("SET", response, ttlResponse); + } + + @Override + public RedisClusterResponse send(JedisCluster jedisCluster, RedisTtl redisTTL) { + instrumentation.logDebug("key: {}, value: {}", key, value); + try { + String response = jedisCluster.set(key, value); + Long ttlResponse = redisTTL.setTtl(jedisCluster, key); + return new RedisClusterResponse("SET", response, ttlResponse); + } catch (JedisException e) { + return new RedisClusterResponse(e.getMessage()); + } + } + + @Override + public String toString() { + return String.format("RedisKeyValueEntry: Key %s, Value %s", key, value); + } +} diff --git a/src/main/java/io/odpf/depot/redis/client/entry/RedisListEntry.java b/src/main/java/io/odpf/depot/redis/client/entry/RedisListEntry.java new file mode 100644 index 00000000..23975404 --- /dev/null +++ b/src/main/java/io/odpf/depot/redis/client/entry/RedisListEntry.java @@ -0,0 +1,49 @@ +package io.odpf.depot.redis.client.entry; + +import io.odpf.depot.metrics.Instrumentation; +import io.odpf.depot.redis.client.response.RedisClusterResponse; +import io.odpf.depot.redis.client.response.RedisStandaloneResponse; +import io.odpf.depot.redis.ttl.RedisTtl; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import redis.clients.jedis.JedisCluster; +import redis.clients.jedis.Pipeline; +import redis.clients.jedis.Response; +import redis.clients.jedis.exceptions.JedisException; + +/** + * Class for Redis Hash set entry. + */ +@AllArgsConstructor +@EqualsAndHashCode +public class RedisListEntry implements RedisEntry { + private final String key; + private final String value; + @EqualsAndHashCode.Exclude + private final Instrumentation instrumentation; + + @Override + public RedisStandaloneResponse send(Pipeline jedisPipelined, RedisTtl redisTTL) { + instrumentation.logDebug("key: {}, value: {}", key, value); + Response response = jedisPipelined.lpush(key, value); + Response ttlResponse = redisTTL.setTtl(jedisPipelined, key); + return new RedisStandaloneResponse("LPUSH", response, ttlResponse); + } + + @Override + public RedisClusterResponse send(JedisCluster jedisCluster, RedisTtl redisTTL) { + instrumentation.logDebug("key: {}, value: {}", key, value); + try { + Long response = jedisCluster.lpush(key, value); + Long ttlResponse = redisTTL.setTtl(jedisCluster, key); + return new RedisClusterResponse("LPUSH", response, ttlResponse); + } catch (JedisException e) { + return new RedisClusterResponse(e.getMessage()); + } + } + + @Override + public String toString() { + return String.format("RedisListEntry: Key %s, Value %s", key, value); + } +} diff --git a/src/main/java/io/odpf/depot/redis/client/response/RedisClusterResponse.java b/src/main/java/io/odpf/depot/redis/client/response/RedisClusterResponse.java new file mode 100644 index 00000000..e245f90a --- /dev/null +++ b/src/main/java/io/odpf/depot/redis/client/response/RedisClusterResponse.java @@ -0,0 +1,24 @@ +package io.odpf.depot.redis.client.response; + +import lombok.Getter; + +public class RedisClusterResponse implements RedisResponse { + @Getter + private final String message; + @Getter + private final boolean failed; + + public RedisClusterResponse(String command, Object response, Long ttlResponse) { + this.message = String.format( + "%s: %s, TTL: %s", + command, + response, + ttlResponse == null ? "NoOp" : ttlResponse == 0 ? "NOT UPDATED" : "UPDATED"); + this.failed = false; + } + + public RedisClusterResponse(String message) { + this.message = message; + this.failed = true; + } +} diff --git a/src/main/java/io/odpf/depot/redis/client/response/RedisResponse.java b/src/main/java/io/odpf/depot/redis/client/response/RedisResponse.java new file mode 100644 index 00000000..4e5c6c28 --- /dev/null +++ b/src/main/java/io/odpf/depot/redis/client/response/RedisResponse.java @@ -0,0 +1,7 @@ +package io.odpf.depot.redis.client.response; + +public interface RedisResponse { + String getMessage(); + + boolean isFailed(); +} diff --git a/src/main/java/io/odpf/depot/redis/client/response/RedisStandaloneResponse.java b/src/main/java/io/odpf/depot/redis/client/response/RedisStandaloneResponse.java new file mode 100644 index 00000000..8791c190 --- /dev/null +++ b/src/main/java/io/odpf/depot/redis/client/response/RedisStandaloneResponse.java @@ -0,0 +1,34 @@ +package io.odpf.depot.redis.client.response; + +import lombok.Getter; +import redis.clients.jedis.Response; +import redis.clients.jedis.exceptions.JedisException; + +public class RedisStandaloneResponse implements RedisResponse { + private final Response response; + private final Response ttlResponse; + private final String command; + @Getter + private String message; + @Getter + private boolean failed = true; + + public RedisStandaloneResponse(String command, Response response, Response ttlResponse) { + this.command = command; + this.response = response; + this.ttlResponse = ttlResponse; + } + + public RedisStandaloneResponse process() { + try { + Object cmd = response.get(); + Object ttl = ttlResponse != null ? (((long) ttlResponse.get()) == 0L ? "NOT UPDATED" : "UPDATED") : "NoOp"; + message = String.format("%s: %s, TTL: %s", command, cmd, ttl); + failed = false; + } catch (JedisException e) { + message = e.getMessage(); + failed = true; + } + return this; + } +} diff --git a/src/main/java/io/odpf/depot/redis/enums/RedisSinkDataType.java b/src/main/java/io/odpf/depot/redis/enums/RedisSinkDataType.java new file mode 100644 index 00000000..23d791a2 --- /dev/null +++ b/src/main/java/io/odpf/depot/redis/enums/RedisSinkDataType.java @@ -0,0 +1,7 @@ +package io.odpf.depot.redis.enums; + +public enum RedisSinkDataType { + LIST, + HASHSET, + KEYVALUE, +} diff --git a/src/main/java/io/odpf/depot/redis/enums/RedisSinkDeploymentType.java b/src/main/java/io/odpf/depot/redis/enums/RedisSinkDeploymentType.java new file mode 100644 index 00000000..85a70bcd --- /dev/null +++ b/src/main/java/io/odpf/depot/redis/enums/RedisSinkDeploymentType.java @@ -0,0 +1,6 @@ +package io.odpf.depot.redis.enums; + +public enum RedisSinkDeploymentType { + STANDALONE, + CLUSTER +} diff --git a/src/main/java/io/odpf/depot/redis/enums/RedisSinkTtlType.java b/src/main/java/io/odpf/depot/redis/enums/RedisSinkTtlType.java new file mode 100644 index 00000000..41d76a4f --- /dev/null +++ b/src/main/java/io/odpf/depot/redis/enums/RedisSinkTtlType.java @@ -0,0 +1,7 @@ +package io.odpf.depot.redis.enums; + +public enum RedisSinkTtlType { + EXACT_TIME, + DURATION, + DISABLE +} diff --git a/src/main/java/io/odpf/depot/redis/parsers/RedisEntryParser.java b/src/main/java/io/odpf/depot/redis/parsers/RedisEntryParser.java new file mode 100644 index 00000000..8efc31dc --- /dev/null +++ b/src/main/java/io/odpf/depot/redis/parsers/RedisEntryParser.java @@ -0,0 +1,11 @@ +package io.odpf.depot.redis.parsers; + +import io.odpf.depot.message.ParsedOdpfMessage; +import io.odpf.depot.redis.client.entry.RedisEntry; + +import java.util.List; + +public interface RedisEntryParser { + + List getRedisEntry(ParsedOdpfMessage parsedOdpfMessage); +} diff --git a/src/main/java/io/odpf/depot/redis/parsers/RedisEntryParserFactory.java b/src/main/java/io/odpf/depot/redis/parsers/RedisEntryParserFactory.java new file mode 100644 index 00000000..8f519b87 --- /dev/null +++ b/src/main/java/io/odpf/depot/redis/parsers/RedisEntryParserFactory.java @@ -0,0 +1,45 @@ +package io.odpf.depot.redis.parsers; + +import io.odpf.depot.config.RedisSinkConfig; +import io.odpf.depot.message.OdpfMessageSchema; +import io.odpf.depot.metrics.StatsDReporter; + +import java.util.Map; +import java.util.Properties; +import java.util.stream.Collectors; + +/** + * Redis parser factory. + */ +public class RedisEntryParserFactory { + + public static RedisEntryParser getRedisEntryParser( + RedisSinkConfig redisSinkConfig, + StatsDReporter statsDReporter, + OdpfMessageSchema schema) { + Template keyTemplate = new Template(redisSinkConfig.getSinkRedisKeyTemplate()); + switch (redisSinkConfig.getSinkRedisDataType()) { + case KEYVALUE: + String fieldName = redisSinkConfig.getSinkRedisKeyValueDataFieldName(); + if (fieldName == null || fieldName.isEmpty()) { + throw new IllegalArgumentException("Empty config SINK_REDIS_KEY_VALUE_DATA_FIELD_NAME found"); + } + return new RedisKeyValueEntryParser(statsDReporter, keyTemplate, fieldName, schema); + case LIST: + String field = redisSinkConfig.getSinkRedisListDataFieldName(); + if (field == null || field.isEmpty()) { + throw new IllegalArgumentException("Empty config SINK_REDIS_LIST_DATA_FIELD_NAME found"); + } + return new RedisListEntryParser(statsDReporter, keyTemplate, field, schema); + default: + Properties properties = redisSinkConfig.getSinkRedisHashsetFieldToColumnMapping(); + if (properties == null || properties.isEmpty()) { + throw new IllegalArgumentException("Empty config SINK_REDIS_HASHSET_FIELD_TO_COLUMN_MAPPING found"); + } + Map fieldTemplates = properties.entrySet().stream().collect(Collectors.toMap( + kv -> kv.getKey().toString(), kv -> new Template(kv.getValue().toString()) + )); + return new RedisHashSetEntryParser(statsDReporter, keyTemplate, fieldTemplates, schema); + } + } +} diff --git a/src/main/java/io/odpf/depot/redis/parsers/RedisHashSetEntryParser.java b/src/main/java/io/odpf/depot/redis/parsers/RedisHashSetEntryParser.java new file mode 100644 index 00000000..74e000f5 --- /dev/null +++ b/src/main/java/io/odpf/depot/redis/parsers/RedisHashSetEntryParser.java @@ -0,0 +1,38 @@ +package io.odpf.depot.redis.parsers; + +import io.odpf.depot.message.OdpfMessageSchema; +import io.odpf.depot.message.ParsedOdpfMessage; +import io.odpf.depot.metrics.Instrumentation; +import io.odpf.depot.metrics.StatsDReporter; +import io.odpf.depot.redis.client.entry.RedisEntry; +import io.odpf.depot.redis.client.entry.RedisHashSetFieldEntry; +import lombok.AllArgsConstructor; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + + +/** + * Redis hash set parser. + */ +@AllArgsConstructor +public class RedisHashSetEntryParser implements RedisEntryParser { + private final StatsDReporter statsDReporter; + private final Template keyTemplate; + private final Map fieldTemplates; + private final OdpfMessageSchema schema; + + @Override + public List getRedisEntry(ParsedOdpfMessage parsedOdpfMessage) { + String redisKey = keyTemplate.parse(parsedOdpfMessage, schema); + return fieldTemplates + .entrySet() + .stream() + .map(fieldTemplate -> { + String field = fieldTemplate.getValue().parse(parsedOdpfMessage, schema); + String redisValue = parsedOdpfMessage.getFieldByName(fieldTemplate.getKey(), schema).toString(); + return new RedisHashSetFieldEntry(redisKey, field, redisValue, new Instrumentation(statsDReporter, RedisHashSetFieldEntry.class)); + }).collect(Collectors.toList()); + } +} diff --git a/src/main/java/io/odpf/depot/redis/parsers/RedisKeyValueEntryParser.java b/src/main/java/io/odpf/depot/redis/parsers/RedisKeyValueEntryParser.java new file mode 100644 index 00000000..51e31b1c --- /dev/null +++ b/src/main/java/io/odpf/depot/redis/parsers/RedisKeyValueEntryParser.java @@ -0,0 +1,28 @@ +package io.odpf.depot.redis.parsers; + +import io.odpf.depot.message.OdpfMessageSchema; +import io.odpf.depot.message.ParsedOdpfMessage; +import io.odpf.depot.metrics.Instrumentation; +import io.odpf.depot.metrics.StatsDReporter; +import io.odpf.depot.redis.client.entry.RedisEntry; +import io.odpf.depot.redis.client.entry.RedisKeyValueEntry; +import lombok.AllArgsConstructor; + +import java.util.Collections; +import java.util.List; + +@AllArgsConstructor +public class RedisKeyValueEntryParser implements RedisEntryParser { + private final StatsDReporter statsDReporter; + private final Template keyTemplate; + private final String fieldName; + private final OdpfMessageSchema schema; + + @Override + public List getRedisEntry(ParsedOdpfMessage parsedOdpfMessage) { + String redisKey = keyTemplate.parse(parsedOdpfMessage, schema); + String redisValue = parsedOdpfMessage.getFieldByName(fieldName, schema).toString(); + RedisKeyValueEntry redisKeyValueEntry = new RedisKeyValueEntry(redisKey, redisValue, new Instrumentation(statsDReporter, RedisKeyValueEntry.class)); + return Collections.singletonList(redisKeyValueEntry); + } +} diff --git a/src/main/java/io/odpf/depot/redis/parsers/RedisListEntryParser.java b/src/main/java/io/odpf/depot/redis/parsers/RedisListEntryParser.java new file mode 100644 index 00000000..3a730746 --- /dev/null +++ b/src/main/java/io/odpf/depot/redis/parsers/RedisListEntryParser.java @@ -0,0 +1,31 @@ +package io.odpf.depot.redis.parsers; + + +import io.odpf.depot.message.OdpfMessageSchema; +import io.odpf.depot.message.ParsedOdpfMessage; +import io.odpf.depot.metrics.Instrumentation; +import io.odpf.depot.metrics.StatsDReporter; +import io.odpf.depot.redis.client.entry.RedisEntry; +import io.odpf.depot.redis.client.entry.RedisListEntry; +import lombok.AllArgsConstructor; + +import java.util.Collections; +import java.util.List; + +/** + * Redis list parser. + */ +@AllArgsConstructor +public class RedisListEntryParser implements RedisEntryParser { + private final StatsDReporter statsDReporter; + private final Template keyTemplate; + private final String field; + private final OdpfMessageSchema schema; + + @Override + public List getRedisEntry(ParsedOdpfMessage parsedOdpfMessage) { + String redisKey = keyTemplate.parse(parsedOdpfMessage, schema); + String redisValue = parsedOdpfMessage.getFieldByName(field, schema).toString(); + return Collections.singletonList(new RedisListEntry(redisKey, redisValue, new Instrumentation(statsDReporter, RedisListEntry.class))); + } +} diff --git a/src/main/java/io/odpf/depot/redis/parsers/RedisParser.java b/src/main/java/io/odpf/depot/redis/parsers/RedisParser.java new file mode 100644 index 00000000..6f383451 --- /dev/null +++ b/src/main/java/io/odpf/depot/redis/parsers/RedisParser.java @@ -0,0 +1,60 @@ +package io.odpf.depot.redis.parsers; + +import io.odpf.depot.common.Tuple; +import io.odpf.depot.error.ErrorInfo; +import io.odpf.depot.error.ErrorType; +import io.odpf.depot.exception.ConfigurationException; +import io.odpf.depot.exception.DeserializerException; +import io.odpf.depot.message.OdpfMessage; +import io.odpf.depot.message.OdpfMessageParser; +import io.odpf.depot.message.ParsedOdpfMessage; +import io.odpf.depot.message.SinkConnectorSchemaMessageMode; +import io.odpf.depot.redis.client.entry.RedisEntry; +import io.odpf.depot.redis.record.RedisRecord; +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.IntStream; + + +/** + * Convert Odpf messages to RedisRecords. + */ + +@AllArgsConstructor +@Slf4j +public class RedisParser { + private final OdpfMessageParser odpfMessageParser; + private final RedisEntryParser redisEntryParser; + private final Tuple modeAndSchema; + + public List convert(List messages) { + List records = new ArrayList<>(); + IntStream.range(0, messages.size()).forEach(index -> { + try { + ParsedOdpfMessage parsedOdpfMessage = odpfMessageParser.parse(messages.get(index), modeAndSchema.getFirst(), modeAndSchema.getSecond()); + List redisDataEntries = redisEntryParser.getRedisEntry(parsedOdpfMessage); + for (RedisEntry redisEntry : redisDataEntries) { + records.add(new RedisRecord(redisEntry, (long) index, null, messages.get(index).getMetadataString(), true)); + } + } catch (ConfigurationException e) { + records.add(createAndLogErrorRecord(e, ErrorType.UNKNOWN_FIELDS_ERROR, index, messages)); + } catch (IllegalArgumentException e) { + records.add(createAndLogErrorRecord(e, ErrorType.DEFAULT_ERROR, index, messages)); + } catch (DeserializerException | IOException e) { + records.add(createAndLogErrorRecord(e, ErrorType.DESERIALIZATION_ERROR, index, messages)); + } + }); + return records; + } + + private RedisRecord createAndLogErrorRecord(Exception e, ErrorType type, int index, List messages) { + ErrorInfo errorInfo = new ErrorInfo(e, type); + RedisRecord record = new RedisRecord(null, (long) index, errorInfo, messages.get(index).getMetadataString(), false); + log.error("Error while parsing record for message. Record: {}, Error: {}", record, errorInfo); + return record; + } +} diff --git a/src/main/java/io/odpf/depot/redis/parsers/Template.java b/src/main/java/io/odpf/depot/redis/parsers/Template.java new file mode 100644 index 00000000..55e605e2 --- /dev/null +++ b/src/main/java/io/odpf/depot/redis/parsers/Template.java @@ -0,0 +1,42 @@ +package io.odpf.depot.redis.parsers; + +import com.google.common.base.Splitter; +import io.odpf.depot.message.OdpfMessageSchema; +import io.odpf.depot.message.ParsedOdpfMessage; +import io.odpf.depot.utils.StringUtils; + +import java.util.ArrayList; +import java.util.List; + +public class Template { + private final String templatePattern; + private final List patternVariableFieldNames; + + public Template(String template) { + if (template == null || template.isEmpty()) { + throw new IllegalArgumentException("Template '" + template + "' is invalid"); + } + List templateStrings = new ArrayList<>(); + Splitter.on(",").omitEmptyStrings().split(template).forEach(s -> templateStrings.add(s.trim())); + this.templatePattern = templateStrings.get(0); + this.patternVariableFieldNames = templateStrings.subList(1, templateStrings.size()); + validate(); + } + + private void validate() { + int validArgs = StringUtils.countVariables(templatePattern); + int values = patternVariableFieldNames.size(); + int variables = StringUtils.count(templatePattern, '%'); + if (validArgs != values || variables != values) { + throw new IllegalArgumentException(String.format("Template is not valid, variables=%d, validArgs=%d, values=%d", variables, validArgs, values)); + } + } + + public String parse(ParsedOdpfMessage parsedOdpfMessage, OdpfMessageSchema schema) { + Object[] patternVariableData = patternVariableFieldNames + .stream() + .map(fieldName -> parsedOdpfMessage.getFieldByName(fieldName, schema)) + .toArray(); + return String.format(templatePattern, patternVariableData); + } +} diff --git a/src/main/java/io/odpf/depot/redis/record/RedisRecord.java b/src/main/java/io/odpf/depot/redis/record/RedisRecord.java new file mode 100644 index 00000000..d526306d --- /dev/null +++ b/src/main/java/io/odpf/depot/redis/record/RedisRecord.java @@ -0,0 +1,38 @@ +package io.odpf.depot.redis.record; + +import io.odpf.depot.error.ErrorInfo; +import io.odpf.depot.redis.client.entry.RedisEntry; +import io.odpf.depot.redis.client.response.RedisClusterResponse; +import io.odpf.depot.redis.client.response.RedisStandaloneResponse; +import io.odpf.depot.redis.ttl.RedisTtl; +import lombok.AllArgsConstructor; +import lombok.Getter; +import redis.clients.jedis.JedisCluster; +import redis.clients.jedis.Pipeline; + + +@AllArgsConstructor +public class RedisRecord { + private RedisEntry redisEntry; + @Getter + private final Long index; + @Getter + private final ErrorInfo errorInfo; + @Getter + private final String metadata; + @Getter + private final boolean valid; + + public RedisStandaloneResponse send(Pipeline jedisPipelined, RedisTtl redisTTL) { + return redisEntry.send(jedisPipelined, redisTTL); + } + + public RedisClusterResponse send(JedisCluster jedisCluster, RedisTtl redisTTL) { + return redisEntry.send(jedisCluster, redisTTL); + } + + @Override + public String toString() { + return String.format("Metadata %s %s", metadata, redisEntry != null ? redisEntry.toString() : "NULL"); + } +} diff --git a/src/main/java/io/odpf/depot/redis/ttl/DurationTtl.java b/src/main/java/io/odpf/depot/redis/ttl/DurationTtl.java new file mode 100644 index 00000000..557056a5 --- /dev/null +++ b/src/main/java/io/odpf/depot/redis/ttl/DurationTtl.java @@ -0,0 +1,22 @@ +package io.odpf.depot.redis.ttl; + +import lombok.AllArgsConstructor; +import redis.clients.jedis.JedisCluster; +import redis.clients.jedis.Pipeline; +import redis.clients.jedis.Response; + + +@AllArgsConstructor +public class DurationTtl implements RedisTtl { + private int ttlInSeconds; + + @Override + public Response setTtl(Pipeline jedisPipelined, String key) { + return jedisPipelined.expire(key, ttlInSeconds); + } + + @Override + public Long setTtl(JedisCluster jedisCluster, String key) { + return jedisCluster.expire(key, ttlInSeconds); + } +} diff --git a/src/main/java/io/odpf/depot/redis/ttl/ExactTimeTtl.java b/src/main/java/io/odpf/depot/redis/ttl/ExactTimeTtl.java new file mode 100644 index 00000000..e678a754 --- /dev/null +++ b/src/main/java/io/odpf/depot/redis/ttl/ExactTimeTtl.java @@ -0,0 +1,22 @@ +package io.odpf.depot.redis.ttl; + +import lombok.AllArgsConstructor; +import redis.clients.jedis.JedisCluster; +import redis.clients.jedis.Pipeline; +import redis.clients.jedis.Response; + + +@AllArgsConstructor +public class ExactTimeTtl implements RedisTtl { + private long unixTime; + + @Override + public Response setTtl(Pipeline jedisPipelined, String key) { + return jedisPipelined.expireAt(key, unixTime); + } + + @Override + public Long setTtl(JedisCluster jedisCluster, String key) { + return jedisCluster.expireAt(key, unixTime); + } +} diff --git a/src/main/java/io/odpf/depot/redis/ttl/NoRedisTtl.java b/src/main/java/io/odpf/depot/redis/ttl/NoRedisTtl.java new file mode 100644 index 00000000..076f45cd --- /dev/null +++ b/src/main/java/io/odpf/depot/redis/ttl/NoRedisTtl.java @@ -0,0 +1,17 @@ +package io.odpf.depot.redis.ttl; + +import redis.clients.jedis.JedisCluster; +import redis.clients.jedis.Pipeline; +import redis.clients.jedis.Response; + +public class NoRedisTtl implements RedisTtl { + @Override + public Response setTtl(Pipeline jedisPipelined, String key) { + return null; + } + + @Override + public Long setTtl(JedisCluster jedisCluster, String key) { + return null; + } +} diff --git a/src/main/java/io/odpf/depot/redis/ttl/RedisTTLFactory.java b/src/main/java/io/odpf/depot/redis/ttl/RedisTTLFactory.java new file mode 100644 index 00000000..f6fb0642 --- /dev/null +++ b/src/main/java/io/odpf/depot/redis/ttl/RedisTTLFactory.java @@ -0,0 +1,27 @@ +package io.odpf.depot.redis.ttl; + + +import io.odpf.depot.config.RedisSinkConfig; +import io.odpf.depot.exception.ConfigurationException; +import io.odpf.depot.redis.enums.RedisSinkTtlType; + +public class RedisTTLFactory { + + public static RedisTtl getTTl(RedisSinkConfig redisSinkConfig) { + if (redisSinkConfig.getSinkRedisTtlType() == RedisSinkTtlType.DISABLE) { + return new NoRedisTtl(); + } + long redisTTLValue = redisSinkConfig.getSinkRedisTtlValue(); + if (redisTTLValue < 0) { + throw new ConfigurationException("Provide a positive TTL value"); + } + switch (redisSinkConfig.getSinkRedisTtlType()) { + case EXACT_TIME: + return new ExactTimeTtl(redisTTLValue); + case DURATION: + return new DurationTtl((int) redisTTLValue); + default: + throw new ConfigurationException("Not a valid TTL config"); + } + } +} diff --git a/src/main/java/io/odpf/depot/redis/ttl/RedisTtl.java b/src/main/java/io/odpf/depot/redis/ttl/RedisTtl.java new file mode 100644 index 00000000..2ebc8292 --- /dev/null +++ b/src/main/java/io/odpf/depot/redis/ttl/RedisTtl.java @@ -0,0 +1,14 @@ +package io.odpf.depot.redis.ttl; + +import redis.clients.jedis.JedisCluster; +import redis.clients.jedis.Pipeline; +import redis.clients.jedis.Response; + +/** + * Interface for RedisTTL. + */ +public interface RedisTtl { + Response setTtl(Pipeline jedisPipelined, String key); + + Long setTtl(JedisCluster jedisCluster, String key); +} diff --git a/src/main/java/io/odpf/depot/redis/util/RedisSinkUtils.java b/src/main/java/io/odpf/depot/redis/util/RedisSinkUtils.java new file mode 100644 index 00000000..466d60ab --- /dev/null +++ b/src/main/java/io/odpf/depot/redis/util/RedisSinkUtils.java @@ -0,0 +1,30 @@ +package io.odpf.depot.redis.util; + +import io.odpf.depot.error.ErrorInfo; +import io.odpf.depot.error.ErrorType; +import io.odpf.depot.metrics.Instrumentation; +import io.odpf.depot.redis.client.response.RedisResponse; +import io.odpf.depot.redis.record.RedisRecord; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.IntStream; + +public class RedisSinkUtils { + public static Map getErrorsFromResponse(List redisRecords, List responses, Instrumentation instrumentation) { + Map errors = new HashMap<>(); + IntStream.range(0, responses.size()).forEach( + index -> { + RedisResponse response = responses.get(index); + if (response.isFailed()) { + RedisRecord record = redisRecords.get(index); + instrumentation.logError("Error while inserting to redis for message. Record: {}, Error: {}", + record.toString(), response.getMessage()); + errors.put(record.getIndex(), new ErrorInfo(new Exception(response.getMessage()), ErrorType.DEFAULT_ERROR)); + } + } + ); + return errors; + } +} diff --git a/src/main/java/io/odpf/depot/utils/JsonUtils.java b/src/main/java/io/odpf/depot/utils/JsonUtils.java new file mode 100644 index 00000000..37c3e3c2 --- /dev/null +++ b/src/main/java/io/odpf/depot/utils/JsonUtils.java @@ -0,0 +1,36 @@ +package io.odpf.depot.utils; + +import io.odpf.depot.config.OdpfSinkConfig; +import org.json.JSONObject; + +public class JsonUtils { + /** + * Creates a json Object based on the configuration. + * If String mode is enabled, it converts all the fields in string. + * + * @param config Sink Configuration + * @param payload Json Payload in byyes + * @return Json object + */ + public static JSONObject getJsonObject(OdpfSinkConfig config, byte[] payload) { + JSONObject jsonObject = new JSONObject(new String(payload)); + if (!config.getSinkConnectorSchemaJsonParserStringModeEnabled()) { + return jsonObject; + } + // convert to all objects to string + JSONObject jsonWithStringValues = new JSONObject(); + jsonObject.keySet() + .forEach(k -> { + Object value = jsonObject.get(k); + if (value instanceof JSONObject) { + throw new UnsupportedOperationException("nested json structure not supported yet"); + } + if (JSONObject.NULL.equals(value)) { + return; + } + jsonWithStringValues.put(k, value.toString()); + }); + + return jsonWithStringValues; + } +} diff --git a/src/main/java/io/odpf/depot/utils/MessageConfigUtils.java b/src/main/java/io/odpf/depot/utils/MessageConfigUtils.java new file mode 100644 index 00000000..fcc3f7eb --- /dev/null +++ b/src/main/java/io/odpf/depot/utils/MessageConfigUtils.java @@ -0,0 +1,15 @@ +package io.odpf.depot.utils; + +import io.odpf.depot.common.Tuple; +import io.odpf.depot.config.OdpfSinkConfig; +import io.odpf.depot.message.SinkConnectorSchemaMessageMode; + +public class MessageConfigUtils { + + public static Tuple getModeAndSchema(OdpfSinkConfig sinkConfig) { + SinkConnectorSchemaMessageMode mode = sinkConfig.getSinkConnectorSchemaMessageMode(); + String schemaClass = mode == SinkConnectorSchemaMessageMode.LOG_MESSAGE + ? sinkConfig.getSinkConnectorSchemaProtoMessageClass() : sinkConfig.getSinkConnectorSchemaProtoKeyClass(); + return new Tuple<>(mode, schemaClass); + } +} diff --git a/src/main/java/io/odpf/depot/utils/StringUtils.java b/src/main/java/io/odpf/depot/utils/StringUtils.java new file mode 100644 index 00000000..9e6a8a6d --- /dev/null +++ b/src/main/java/io/odpf/depot/utils/StringUtils.java @@ -0,0 +1,37 @@ +package io.odpf.depot.utils; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.IntStream; + +public class StringUtils { + + private static final Pattern PATTERN = Pattern.compile("(?!<%)%" + + "(?:(\\d+)\\$)?" + + "([-#+ 0,(]|<)?" + + "\\d*" + + "(?:\\.\\d+)?" + + "(?:[bBhHsScCdoxXeEfgGaAtT]|" + + "[tT][HIklMSLNpzZsQBbhAaCYyjmdeRTrDFc])"); + + public static int countVariables(String fmt) { + Matcher m = PATTERN.matcher(fmt); + int np = 0; + int maxref = 0; + while (m.find()) { + if (m.group(1) != null) { + String dec = m.group(1); + int ref = Integer.parseInt(dec); + maxref = Math.max(ref, maxref); + } else if (!(m.group(2) != null && "<".equals(m.group(2)))) { + np++; + } + } + return Math.max(np, maxref); + } + + public static int count(String in, char c) { + return IntStream.range(0, in.length()). + reduce(0, (x, y) -> x + (in.charAt(y) == c ? 1 : 0)); + } +} diff --git a/src/test/java/io/odpf/depot/config/RedisSinkConfigTest.java b/src/test/java/io/odpf/depot/config/RedisSinkConfigTest.java new file mode 100644 index 00000000..6b31c7fe --- /dev/null +++ b/src/test/java/io/odpf/depot/config/RedisSinkConfigTest.java @@ -0,0 +1,20 @@ +package io.odpf.depot.config; + +import io.odpf.depot.redis.enums.RedisSinkDeploymentType; +import io.odpf.depot.redis.enums.RedisSinkTtlType; +import org.aeonbits.owner.ConfigFactory; +import org.junit.Assert; +import org.junit.Test; + +public class RedisSinkConfigTest { + @Test + public void testMetadataTypes() { + System.setProperty("SINK_REDIS_DEPLOYMENT_TYPE", "standalone"); + System.setProperty("SINK_REDIS_TTL_TYPE", "disable"); + System.setProperty("SINK_REDIS_KEY_TEMPLATE", "test-key"); + RedisSinkConfig config = ConfigFactory.create(RedisSinkConfig.class, System.getProperties()); + Assert.assertEquals("test-key", config.getSinkRedisKeyTemplate()); + Assert.assertEquals(RedisSinkDeploymentType.STANDALONE, config.getSinkRedisDeploymentType()); + Assert.assertEquals(RedisSinkTtlType.DISABLE, config.getSinkRedisTtlType()); + } +} diff --git a/src/test/java/io/odpf/depot/config/converter/JsonToPropertiesConverterTest.java b/src/test/java/io/odpf/depot/config/converter/JsonToPropertiesConverterTest.java new file mode 100644 index 00000000..7e8269d8 --- /dev/null +++ b/src/test/java/io/odpf/depot/config/converter/JsonToPropertiesConverterTest.java @@ -0,0 +1,72 @@ +package io.odpf.depot.config.converter; + +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.util.Properties; +import static org.junit.Assert.*; + +public class JsonToPropertiesConverterTest { + + @Rule + public ExpectedException thrown = ExpectedException.none(); + + @Test + public void shouldConvertJSONConfigToProperties() { + String json = "{\"order_number\":\"ORDER_NUMBER\",\"event_timestamp\":\"TIMESTAMP\",\"driver_id\":\"DRIVER_ID\"}"; + + Properties properties = new JsonToPropertiesConverter().convert(null, json); + + assertEquals(3, properties.size()); + assertEquals("ORDER_NUMBER", properties.get("order_number")); + assertEquals("TIMESTAMP", properties.get("event_timestamp")); + assertEquals("DRIVER_ID", properties.get("driver_id")); + } + + @Test + public void shouldValidateJsonConfigForDuplicates() { + String json = "{\"order_number\":\"ORDER_NUMBER\",\"event_timestamp\":\"TIMESTAMP\",\"driver_id\":\"TIMESTAMP\"}"; + IllegalArgumentException e = Assert.assertThrows(IllegalArgumentException.class, () -> new JsonToPropertiesConverter().convert(null, json)); + Assert.assertEquals("duplicates found in SINK_REDIS_HASHSET_FIELD_TO_COLUMN_MAPPING for : [TIMESTAMP]", e.getMessage()); + } + + @Test + public void shouldValidateJsonConfigForDuplicatesInNestedJsons() { + String json = "{\"order_number\":\"ORDER_NUMBER\",\"event_timestamp\":\"TIMESTAMP\",\"nested\":{\"1\":\"TIMESTAMP\",\"2\":\"ORDER_NUMBER\"}}"; + IllegalArgumentException e = Assert.assertThrows(IllegalArgumentException.class, () -> new JsonToPropertiesConverter().convert(null, json)); + Assert.assertEquals("duplicates found in SINK_REDIS_HASHSET_FIELD_TO_COLUMN_MAPPING for : [ORDER_NUMBER, TIMESTAMP]", e.getMessage()); + } + + @Test + public void shouldConvertNestedJSONToNestedProperties() { + String json = "{\"order_id\":{\"order_number\":\"ORDER_NUMBER\",\"order_url\":\"ORDER_URL\",\"order_details\":\"ORDER_DETAILS\"},\"nested_order_details\":\"NUMBER_FIELDS\"}"; + + Properties actualProperties = new JsonToPropertiesConverter().convert(null, json); + + Properties expectedNestedProperties = new Properties(); + expectedNestedProperties.put("order_number", "ORDER_NUMBER"); + expectedNestedProperties.put("order_url", "ORDER_URL"); + expectedNestedProperties.put("order_details", "ORDER_DETAILS"); + + Properties expectedProperties = new Properties(); + expectedProperties.put("order_id", expectedNestedProperties); + expectedProperties.put("nested_order_details", "NUMBER_FIELDS"); + + assertEquals(actualProperties, expectedProperties); + } + + @Test + public void shouldNotProcessEmptyStringAsProperties() { + String json = ""; + Properties actualProperties = new JsonToPropertiesConverter().convert(null, json); + assertNull(actualProperties); + } + + @Test + public void shouldNotProcessNullStringAsProperties() { + Properties actualProperties = new JsonToPropertiesConverter().convert(null, null); + assertNull(actualProperties); + } +} diff --git a/src/test/java/io/odpf/depot/config/converter/RedisSinkDataTypeConverterTest.java b/src/test/java/io/odpf/depot/config/converter/RedisSinkDataTypeConverterTest.java new file mode 100644 index 00000000..b66d2d32 --- /dev/null +++ b/src/test/java/io/odpf/depot/config/converter/RedisSinkDataTypeConverterTest.java @@ -0,0 +1,57 @@ +package io.odpf.depot.config.converter; + +import io.odpf.depot.redis.enums.RedisSinkDataType; +import org.gradle.internal.impldep.org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class RedisSinkDataTypeConverterTest { + + private RedisSinkDataTypeConverter redisSinkDataTypeConverter; + + @Before + public void setUp() { + redisSinkDataTypeConverter = new RedisSinkDataTypeConverter(); + } + + @Test + public void shouldReturnListSinkTypeFromLowerCaseInput() { + RedisSinkDataType redisSinkDataType = redisSinkDataTypeConverter.convert(null, "list"); + Assert.assertTrue(redisSinkDataType.equals(RedisSinkDataType.LIST)); + } + + @Test + public void shouldReturnListSinkTypeFromUpperCaseInput() { + RedisSinkDataType redisSinkDataType = redisSinkDataTypeConverter.convert(null, "LIST"); + Assert.assertTrue(redisSinkDataType.equals(RedisSinkDataType.LIST)); + } + + @Test + public void shouldReturnListSinkTypeFromMixedCaseInput() { + RedisSinkDataType redisSinkDataType = redisSinkDataTypeConverter.convert(null, "LiSt"); + Assert.assertTrue(redisSinkDataType.equals(RedisSinkDataType.LIST)); + } + + @Test + public void shouldReturnHashSetSinkTypeFromInput() { + RedisSinkDataType redisSinkDataType = redisSinkDataTypeConverter.convert(null, "hashset"); + Assert.assertTrue(redisSinkDataType.equals(RedisSinkDataType.HASHSET)); + } + + @Test + public void shouldReturnKeyValueSinkTypeFromInput() { + RedisSinkDataType redisSinkDataType = redisSinkDataTypeConverter.convert(null, "keyvalue"); + Assert.assertTrue(redisSinkDataType.equals(RedisSinkDataType.KEYVALUE)); + } + + @Test(expected = IllegalArgumentException.class) + public void shouldThrowOnEmptyArgument() { + redisSinkDataTypeConverter.convert(null, ""); + } + + @Test(expected = IllegalArgumentException.class) + public void shouldThrowOnInvalidArgument() { + redisSinkDataTypeConverter.convert(null, "INVALID"); + } + +} diff --git a/src/test/java/io/odpf/depot/config/converter/RedisSinkDeploymentTypeConverterTest.java b/src/test/java/io/odpf/depot/config/converter/RedisSinkDeploymentTypeConverterTest.java new file mode 100644 index 00000000..387cab3b --- /dev/null +++ b/src/test/java/io/odpf/depot/config/converter/RedisSinkDeploymentTypeConverterTest.java @@ -0,0 +1,49 @@ +package io.odpf.depot.config.converter; + +import io.odpf.depot.redis.enums.RedisSinkDeploymentType; +import org.gradle.internal.impldep.org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class RedisSinkDeploymentTypeConverterTest { + private RedisSinkDeploymentTypeConverter redisSinkDeploymentTypeConverter; + + @Before + public void setup() { + redisSinkDeploymentTypeConverter = new RedisSinkDeploymentTypeConverter(); + } + + @Test + public void shouldReturnStandaloneTypeFromLowerCaseInput() { + RedisSinkDeploymentType redisSinkDeploymentType = redisSinkDeploymentTypeConverter.convert(null, "standalone"); + Assert.assertTrue(redisSinkDeploymentType.equals(RedisSinkDeploymentType.STANDALONE)); + } + + @Test + public void shouldReturnStandaloneTypeFromUpperCaseInput() { + RedisSinkDeploymentType redisSinkDeploymentType = redisSinkDeploymentTypeConverter.convert(null, "STANDALONE"); + Assert.assertTrue(redisSinkDeploymentType.equals(RedisSinkDeploymentType.STANDALONE)); + } + + @Test + public void shouldReturnStandaloneTypeFromMixedCaseInput() { + RedisSinkDeploymentType redisSinkDeploymentType = redisSinkDeploymentTypeConverter.convert(null, "stANdAlOne"); + Assert.assertTrue(redisSinkDeploymentType.equals(RedisSinkDeploymentType.STANDALONE)); + } + + @Test + public void shouldReturnClusterTypeFromUpperCaseInput() { + RedisSinkDeploymentType redisSinkDeploymentType = redisSinkDeploymentTypeConverter.convert(null, "CLUSTER"); + Assert.assertTrue(redisSinkDeploymentType.equals(RedisSinkDeploymentType.CLUSTER)); + } + + @Test(expected = IllegalArgumentException.class) + public void shouldThrowOnEmptyArgument() { + redisSinkDeploymentTypeConverter.convert(null, ""); + } + + @Test(expected = IllegalArgumentException.class) + public void shouldThrowOnInvalidArgument() { + redisSinkDeploymentTypeConverter.convert(null, "INVALID"); + } +} diff --git a/src/test/java/io/odpf/depot/config/converter/RedisSinkTtlTypeConverterTest.java b/src/test/java/io/odpf/depot/config/converter/RedisSinkTtlTypeConverterTest.java new file mode 100644 index 00000000..50540775 --- /dev/null +++ b/src/test/java/io/odpf/depot/config/converter/RedisSinkTtlTypeConverterTest.java @@ -0,0 +1,55 @@ +package io.odpf.depot.config.converter; + +import io.odpf.depot.redis.enums.RedisSinkTtlType; +import org.gradle.internal.impldep.org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class RedisSinkTtlTypeConverterTest { + private RedisSinkTtlTypeConverter redisSinkTtlTypeConverter; + + @Before + public void setUp() { + redisSinkTtlTypeConverter = new RedisSinkTtlTypeConverter(); + } + + @Test + public void shouldReturnExactTimeTypeFromLowerCaseInput() { + RedisSinkTtlType redisSinkTtlType = redisSinkTtlTypeConverter.convert(null, "exact_time"); + Assert.assertTrue(redisSinkTtlType.equals(RedisSinkTtlType.EXACT_TIME)); + } + + @Test + public void shouldReturnExactTimeTypeFromUpperCaseInput() { + RedisSinkTtlType redisSinkTtlType = redisSinkTtlTypeConverter.convert(null, "EXACT_TIME"); + Assert.assertTrue(redisSinkTtlType.equals(RedisSinkTtlType.EXACT_TIME)); + } + + @Test + public void shouldReturnExactTimeTypeFromMixedCaseInput() { + RedisSinkTtlType redisSinkTtlType = redisSinkTtlTypeConverter.convert(null, "eXAct_TiMe"); + Assert.assertTrue(redisSinkTtlType.equals(RedisSinkTtlType.EXACT_TIME)); + } + + @Test + public void shouldReturnDisableTypeFromInput() { + RedisSinkTtlType redisSinkTtlType = redisSinkTtlTypeConverter.convert(null, "DISABLE"); + Assert.assertTrue(redisSinkTtlType.equals(RedisSinkTtlType.DISABLE)); + } + + @Test + public void shouldReturnDurationTypeFromInput() { + RedisSinkTtlType redisSinkTtlType = redisSinkTtlTypeConverter.convert(null, "DURATION"); + Assert.assertTrue(redisSinkTtlType.equals(RedisSinkTtlType.DURATION)); + } + + @Test(expected = IllegalArgumentException.class) + public void shouldThrowOnEmptyArgument() { + redisSinkTtlTypeConverter.convert(null, ""); + } + + @Test(expected = IllegalArgumentException.class) + public void shouldThrowOnInvalidArgument() { + redisSinkTtlTypeConverter.convert(null, "INVALID"); + } +} diff --git a/src/test/java/io/odpf/depot/message/json/JsonOdpfParsedMessageTest.java b/src/test/java/io/odpf/depot/message/json/JsonOdpfParsedMessageTest.java index 92524c97..3e790c9c 100644 --- a/src/test/java/io/odpf/depot/message/json/JsonOdpfParsedMessageTest.java +++ b/src/test/java/io/odpf/depot/message/json/JsonOdpfParsedMessageTest.java @@ -1,6 +1,9 @@ package io.odpf.depot.message.json; +import com.jayway.jsonpath.JsonPathException; +import org.json.JSONArray; import org.json.JSONObject; +import org.junit.Assert; import org.junit.Test; import java.util.Collections; @@ -36,4 +39,48 @@ public void shouldGetMappings() { expectedMap.put("address", "planet earth"); assertEquals(expectedMap, parsedMessageMapping); } + + @Test + public void shouldReturnValueFromFlatJson() { + JSONObject personDetails = new JSONObject("{\"first_name\": \"john doe\", \"address\": \"planet earth\"}"); + JsonOdpfParsedMessage parsedMessage = new JsonOdpfParsedMessage(personDetails); + Assert.assertEquals("john doe", parsedMessage.getFieldByName("first_name", null)); + } + + @Test + public void shouldReturnValueFromNestedJson() { + JSONObject personDetails = new JSONObject("" + + "{\"first_name\": \"john doe\"," + + " \"address\": \"planet earth\", " + + "\"family\" : {\"brother\" : \"david doe\"}" + + "}"); + JsonOdpfParsedMessage parsedMessage = new JsonOdpfParsedMessage(personDetails); + Assert.assertEquals("david doe", parsedMessage.getFieldByName("family.brother", null)); + } + + @Test + public void shouldThrowExceptionIfNotFound() { + JSONObject personDetails = new JSONObject("" + + "{\"first_name\": \"john doe\"," + + " \"address\": \"planet earth\", " + + "\"family\" : {\"brother\" : \"david doe\"}" + + "}"); + JsonOdpfParsedMessage parsedMessage = new JsonOdpfParsedMessage(personDetails); + JsonPathException jsonPathException = Assert.assertThrows(JsonPathException.class, () -> parsedMessage.getFieldByName("family.sister", null)); + Assert.assertEquals("No results for path: $['family']['sister']", jsonPathException.getMessage()); + } + + @Test + public void shouldReturnListFromNestedJson() { + JSONObject personDetails = new JSONObject("" + + "{\"first_name\": \"john doe\"," + + " \"address\": \"planet earth\", " + + "\"family\" : [{\"brother\" : \"david doe\"}, {\"brother\" : \"cain doe\"}]" + + "}"); + JsonOdpfParsedMessage parsedMessage = new JsonOdpfParsedMessage(personDetails); + JSONArray family = (JSONArray) parsedMessage.getFieldByName("family", null); + Assert.assertEquals(2, family.length()); + Assert.assertEquals("david doe", ((JSONObject) family.get(0)).get("brother")); + Assert.assertEquals("cain doe", ((JSONObject) family.get(1)).get("brother")); + } } diff --git a/src/test/java/io/odpf/depot/message/proto/ProtoOdpfParsedMessageTest.java b/src/test/java/io/odpf/depot/message/proto/ProtoOdpfParsedMessageTest.java index 6cd70738..f4eee52b 100644 --- a/src/test/java/io/odpf/depot/message/proto/ProtoOdpfParsedMessageTest.java +++ b/src/test/java/io/odpf/depot/message/proto/ProtoOdpfParsedMessageTest.java @@ -1,18 +1,11 @@ package io.odpf.depot.message.proto; import com.google.api.client.util.DateTime; -import com.google.protobuf.Descriptors; -import com.google.protobuf.DynamicMessage; -import com.google.protobuf.ListValue; -import com.google.protobuf.Struct; -import com.google.protobuf.Timestamp; -import com.google.protobuf.Value; -import io.odpf.depot.StatusBQ; -import io.odpf.depot.TestKeyBQ; -import io.odpf.depot.TestMessageBQ; -import io.odpf.depot.TestNestedMessageBQ; -import io.odpf.depot.TestNestedRepeatedMessageBQ; +import com.google.protobuf.*; +import io.odpf.depot.*; +import io.odpf.depot.exception.ConfigurationException; import io.odpf.depot.message.OdpfMessageSchema; +import io.odpf.depot.message.ParsedOdpfMessage; import io.odpf.stencil.Parser; import io.odpf.stencil.StencilClientFactory; import io.odpf.stencil.client.StencilClient; @@ -29,9 +22,7 @@ import java.util.List; import java.util.Map; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; +import static org.junit.Assert.*; public class ProtoOdpfParsedMessageTest { @@ -297,8 +288,6 @@ public void shouldParseStructField() throws IOException { @Test public void shouldParseRepeatableStructField() throws IOException { - Value val = Value.newBuilder().setStringValue("test").build(); - TestMessageBQ message = TestMessageBQ.newBuilder() .addAttributes(Struct.newBuilder().putFields("name", Value.newBuilder().setStringValue("John").build()) .putFields("age", Value.newBuilder().setStringValue("50").build()).build()) @@ -309,7 +298,56 @@ public void shouldParseRepeatableStructField() throws IOException { Parser protoParser = StencilClientFactory.getClient().getParser(TestMessageBQ.class.getName()); OdpfMessageSchema odpfMessageSchema = odpfMessageParser.getSchema("io.odpf.depot.TestMessageBQ", descriptorsMap); Map fields = new ProtoOdpfParsedMessage(protoParser.parse(message.toByteArray())).getMapping(odpfMessageSchema); - assertEquals(Arrays.asList("{\"name\":\"John\",\"age\":\"50\"}", "{\"name\":\"John\",\"age\":\"60\"}"), fields.get("attributes")); } + + @Test + public void shouldCacheMappingForSameSchema() throws IOException { + Parser protoParser = StencilClientFactory.getClient().getParser(TestMessageBQ.class.getName()); + TestMessageBQ message = TestMessageBQ.newBuilder() + .addAttributes(Struct.newBuilder().putFields("name", Value.newBuilder().setStringValue("John").build()) + .putFields("age", Value.newBuilder().setStringValue("50").build()).build()) + .addAttributes(Struct.newBuilder().putFields("name", Value.newBuilder().setStringValue("John").build()) + .putFields("age", Value.newBuilder().setStringValue("60").build()).build()) + .build(); + OdpfMessageSchema odpfMessageSchema1 = odpfMessageParser.getSchema("io.odpf.depot.TestMessageBQ", descriptorsMap); + OdpfMessageSchema odpfMessageSchema2 = odpfMessageParser.getSchema("io.odpf.depot.TestMessageBQ", descriptorsMap); + ParsedOdpfMessage parsedOdpfMessage = new ProtoOdpfParsedMessage(protoParser.parse(message.toByteArray())); + Map map1 = parsedOdpfMessage.getMapping(odpfMessageSchema1); + Map map2 = parsedOdpfMessage.getMapping(odpfMessageSchema2); + assertEquals(map1, map2); + } + + + @Test + public void shouldGetFieldByName() throws IOException { + OdpfMessageSchema odpfMessageSchema = odpfMessageParser.getSchema("io.odpf.depot.TestMessageBQ", descriptorsMap); + ProtoOdpfParsedMessage protoOdpfParsedMessage = new ProtoOdpfParsedMessage(dynamicMessage); + Assert.assertEquals("order-1", protoOdpfParsedMessage.getFieldByName("order_number", odpfMessageSchema)); + } + + + @Test + public void shouldGetFieldByNameFromNested() throws IOException { + TestMessageBQ message1 = TestProtoUtil.generateTestMessage(now); + Parser protoParser = StencilClientFactory.getClient().getParser(TestNestedMessageBQ.class.getName()); + OdpfMessageSchema odpfMessageSchema = odpfMessageParser.getSchema("io.odpf.depot.TestNestedMessageBQ", descriptorsMap); + TestNestedMessageBQ nestedMessage = TestNestedMessageBQ.newBuilder().setNestedId("test").setSingleMessage(message1).build(); + ProtoOdpfParsedMessage protoOdpfParsedMessage = new ProtoOdpfParsedMessage(protoParser.parse(nestedMessage.toByteArray())); + Assert.assertEquals("test", protoOdpfParsedMessage.getFieldByName("nested_id", odpfMessageSchema)); + Assert.assertEquals(message1.getOrderNumber(), protoOdpfParsedMessage.getFieldByName("single_message.order_number", odpfMessageSchema)); + } + + + @Test + public void shouldThrowExceptionIfColumnIsNotPresentInProto() throws IOException { + TestMessageBQ message1 = TestProtoUtil.generateTestMessage(now); + Parser protoParser = StencilClientFactory.getClient().getParser(TestNestedMessageBQ.class.getName()); + OdpfMessageSchema odpfMessageSchema = odpfMessageParser.getSchema("io.odpf.depot.TestNestedMessageBQ", descriptorsMap); + TestNestedMessageBQ nestedMessage = TestNestedMessageBQ.newBuilder().setNestedId("test").setSingleMessage(message1).build(); + ProtoOdpfParsedMessage protoOdpfParsedMessage = new ProtoOdpfParsedMessage(protoParser.parse(nestedMessage.toByteArray())); + Assert.assertEquals("test", protoOdpfParsedMessage.getFieldByName("nested_id", odpfMessageSchema)); + ConfigurationException configurationException = assertThrows(ConfigurationException.class, () -> protoOdpfParsedMessage.getFieldByName("single_message.order_id", odpfMessageSchema)); + Assert.assertEquals("Invalid field config : single_message.order_id", configurationException.getMessage()); + } } diff --git a/src/test/java/io/odpf/depot/redis/RedisSinkTest.java b/src/test/java/io/odpf/depot/redis/RedisSinkTest.java new file mode 100644 index 00000000..cc45e9d7 --- /dev/null +++ b/src/test/java/io/odpf/depot/redis/RedisSinkTest.java @@ -0,0 +1,148 @@ +package io.odpf.depot.redis; + +import io.odpf.depot.OdpfSinkResponse; +import io.odpf.depot.error.ErrorInfo; +import io.odpf.depot.error.ErrorType; +import io.odpf.depot.exception.ConfigurationException; +import io.odpf.depot.message.OdpfMessage; +import io.odpf.depot.metrics.Instrumentation; +import io.odpf.depot.redis.client.RedisClient; +import io.odpf.depot.redis.client.entry.RedisListEntry; +import io.odpf.depot.redis.client.response.RedisResponse; +import io.odpf.depot.redis.parsers.RedisParser; +import io.odpf.depot.redis.record.RedisRecord; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class RedisSinkTest { + @Mock + private RedisClient redisClient; + @Mock + private RedisParser redisParser; + @Mock + private Instrumentation instrumentation; + + @Test + public void shouldPushToSink() { + List messages = new ArrayList<>(); + List records = new ArrayList<>(); + records.add(new RedisRecord(new RedisListEntry("key1", "val1", null), 0L, null, null, true)); + records.add(new RedisRecord(new RedisListEntry("key1", "val1", null), 1L, null, null, true)); + records.add(new RedisRecord(new RedisListEntry("key1", "val1", null), 2L, null, null, true)); + records.add(new RedisRecord(new RedisListEntry("key1", "val1", null), 3L, null, null, true)); + records.add(new RedisRecord(new RedisListEntry("key1", "val1", null), 4L, null, null, true)); + List responses = new ArrayList<>(); + responses.add(Mockito.mock(RedisResponse.class)); + responses.add(Mockito.mock(RedisResponse.class)); + responses.add(Mockito.mock(RedisResponse.class)); + responses.add(Mockito.mock(RedisResponse.class)); + responses.add(Mockito.mock(RedisResponse.class)); + when(redisParser.convert(messages)).thenReturn(records); + when(redisClient.send(records)).thenReturn(responses); + RedisSink redisSink = new RedisSink(redisClient, redisParser, instrumentation); + OdpfSinkResponse odpfSinkResponse = redisSink.pushToSink(messages); + Assert.assertFalse(odpfSinkResponse.hasErrors()); + } + + @Test + public void shouldReportParsingErrors() { + List messages = new ArrayList<>(); + List records = new ArrayList<>(); + records.add(new RedisRecord(null, 0L, new ErrorInfo(new IOException(""), ErrorType.DESERIALIZATION_ERROR), null, false)); + records.add(new RedisRecord(new RedisListEntry("key1", "val1", null), 1L, null, null, true)); + records.add(new RedisRecord(null, 2L, new ErrorInfo(new ConfigurationException(""), ErrorType.DEFAULT_ERROR), null, false)); + records.add(new RedisRecord(new RedisListEntry("key1", "val1", null), 3L, null, null, true)); + records.add(new RedisRecord(new RedisListEntry("key1", "val1", null), 4L, null, null, true)); + List responses = new ArrayList<>(); + responses.add(Mockito.mock(RedisResponse.class)); + responses.add(Mockito.mock(RedisResponse.class)); + responses.add(Mockito.mock(RedisResponse.class)); + when(redisParser.convert(messages)).thenReturn(records); + List validRecords = records.stream().filter(RedisRecord::isValid).collect(Collectors.toList()); + when(redisClient.send(validRecords)).thenReturn(responses); + RedisSink redisSink = new RedisSink(redisClient, redisParser, instrumentation); + OdpfSinkResponse odpfSinkResponse = redisSink.pushToSink(messages); + Assert.assertTrue(odpfSinkResponse.hasErrors()); + Assert.assertEquals(2, odpfSinkResponse.getErrors().size()); + Assert.assertEquals(ErrorType.DESERIALIZATION_ERROR, odpfSinkResponse.getErrorsFor(0).getErrorType()); + Assert.assertEquals(ErrorType.DEFAULT_ERROR, odpfSinkResponse.getErrorsFor(2).getErrorType()); + } + + @Test + public void shouldReportClientErrors() { + List messages = new ArrayList<>(); + List records = new ArrayList<>(); + records.add(new RedisRecord(new RedisListEntry("key1", "val1", null), 0L, null, null, true)); + records.add(new RedisRecord(new RedisListEntry("key1", "val1", null), 1L, null, null, true)); + records.add(new RedisRecord(new RedisListEntry("key1", "val1", null), 2L, null, null, true)); + records.add(new RedisRecord(new RedisListEntry("key1", "val1", null), 3L, null, null, true)); + records.add(new RedisRecord(new RedisListEntry("key1", "val1", null), 4L, null, null, true)); + List responses = new ArrayList<>(); + responses.add(Mockito.mock(RedisResponse.class)); + responses.add(Mockito.mock(RedisResponse.class)); + responses.add(Mockito.mock(RedisResponse.class)); + responses.add(Mockito.mock(RedisResponse.class)); + responses.add(Mockito.mock(RedisResponse.class)); + when(responses.get(2).isFailed()).thenReturn(true); + when(responses.get(2).getMessage()).thenReturn("failed at 2"); + when(responses.get(3).isFailed()).thenReturn(true); + when(responses.get(3).getMessage()).thenReturn("failed at 3"); + when(responses.get(4).isFailed()).thenReturn(true); + when(responses.get(4).getMessage()).thenReturn("failed at 4"); + when(redisParser.convert(messages)).thenReturn(records); + List validRecords = records.stream().filter(RedisRecord::isValid).collect(Collectors.toList()); + when(redisClient.send(validRecords)).thenReturn(responses); + when(redisClient.send(records)).thenReturn(responses); + RedisSink redisSink = new RedisSink(redisClient, redisParser, instrumentation); + OdpfSinkResponse odpfSinkResponse = redisSink.pushToSink(messages); + Assert.assertTrue(odpfSinkResponse.hasErrors()); + Assert.assertEquals(3, odpfSinkResponse.getErrors().size()); + Assert.assertEquals(ErrorType.DEFAULT_ERROR, odpfSinkResponse.getErrorsFor(2).getErrorType()); + Assert.assertEquals(ErrorType.DEFAULT_ERROR, odpfSinkResponse.getErrorsFor(3).getErrorType()); + Assert.assertEquals(ErrorType.DEFAULT_ERROR, odpfSinkResponse.getErrorsFor(4).getErrorType()); + Assert.assertEquals("failed at 2", odpfSinkResponse.getErrorsFor(2).getException().getMessage()); + Assert.assertEquals("failed at 3", odpfSinkResponse.getErrorsFor(3).getException().getMessage()); + Assert.assertEquals("failed at 4", odpfSinkResponse.getErrorsFor(4).getException().getMessage()); + } + + @Test + public void shouldReportNetErrors() { + List messages = new ArrayList<>(); + List records = new ArrayList<>(); + records.add(new RedisRecord(null, 0L, new ErrorInfo(new IOException(""), ErrorType.DESERIALIZATION_ERROR), null, false)); + records.add(new RedisRecord(new RedisListEntry("key1", "val1", null), 1L, null, null, true)); + records.add(new RedisRecord(null, 2L, new ErrorInfo(new ConfigurationException(""), ErrorType.DEFAULT_ERROR), null, false)); + records.add(new RedisRecord(new RedisListEntry("key1", "val1", null), 3L, null, null, true)); + records.add(new RedisRecord(new RedisListEntry("key1", "val1", null), 4L, null, null, true)); + List responses = new ArrayList<>(); + responses.add(Mockito.mock(RedisResponse.class)); + responses.add(Mockito.mock(RedisResponse.class)); + responses.add(Mockito.mock(RedisResponse.class)); + when(responses.get(1).isFailed()).thenReturn(true); + when(responses.get(1).getMessage()).thenReturn("failed at 3"); + when(responses.get(2).isFailed()).thenReturn(true); + when(responses.get(2).getMessage()).thenReturn("failed at 4"); + when(redisParser.convert(messages)).thenReturn(records); + List validRecords = records.stream().filter(RedisRecord::isValid).collect(Collectors.toList()); + when(redisClient.send(validRecords)).thenReturn(responses); + RedisSink redisSink = new RedisSink(redisClient, redisParser, instrumentation); + OdpfSinkResponse odpfSinkResponse = redisSink.pushToSink(messages); + Assert.assertEquals(4, odpfSinkResponse.getErrors().size()); + Assert.assertEquals(ErrorType.DESERIALIZATION_ERROR, odpfSinkResponse.getErrorsFor(0).getErrorType()); + Assert.assertEquals(ErrorType.DEFAULT_ERROR, odpfSinkResponse.getErrorsFor(2).getErrorType()); + Assert.assertEquals("failed at 3", odpfSinkResponse.getErrorsFor(3).getException().getMessage()); + Assert.assertEquals("failed at 4", odpfSinkResponse.getErrorsFor(4).getException().getMessage()); + } +} diff --git a/src/test/java/io/odpf/depot/redis/client/RedisClientFactoryTest.java b/src/test/java/io/odpf/depot/redis/client/RedisClientFactoryTest.java new file mode 100644 index 00000000..4c52d313 --- /dev/null +++ b/src/test/java/io/odpf/depot/redis/client/RedisClientFactoryTest.java @@ -0,0 +1,99 @@ +package io.odpf.depot.redis.client; + + + +import io.odpf.depot.config.RedisSinkConfig; +import io.odpf.depot.exception.ConfigurationException; +import io.odpf.depot.metrics.StatsDReporter; +import io.odpf.depot.redis.enums.RedisSinkDeploymentType; +import io.odpf.depot.redis.enums.RedisSinkTtlType; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class RedisClientFactoryTest { + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + @Mock + private RedisSinkConfig redisSinkConfig; + + @Mock + private StatsDReporter statsDReporter; + + @Test + public void shouldGetStandaloneClient() { + when(redisSinkConfig.getSinkRedisTtlType()).thenReturn(RedisSinkTtlType.DURATION); + when(redisSinkConfig.getSinkRedisDeploymentType()).thenReturn(RedisSinkDeploymentType.STANDALONE); + when(redisSinkConfig.getSinkRedisUrls()).thenReturn("0.0.0.0:0"); + + RedisClient client = RedisClientFactory.getClient(redisSinkConfig, statsDReporter); + + Assert.assertEquals(RedisStandaloneClient.class, client.getClass()); + } + + @Test + public void shouldGetStandaloneClientWhenURLHasSpaces() { + when(redisSinkConfig.getSinkRedisTtlType()).thenReturn(RedisSinkTtlType.DURATION); + when(redisSinkConfig.getSinkRedisDeploymentType()).thenReturn(RedisSinkDeploymentType.STANDALONE); + when(redisSinkConfig.getSinkRedisUrls()).thenReturn(" 0.0.0.0:0 "); + + RedisClient client = RedisClientFactory.getClient(redisSinkConfig, statsDReporter); + + Assert.assertEquals(RedisStandaloneClient.class, client.getClass()); + } + + @Test + public void shouldGetClusterClient() { + when(redisSinkConfig.getSinkRedisTtlType()).thenReturn(RedisSinkTtlType.DURATION); + when(redisSinkConfig.getSinkRedisDeploymentType()).thenReturn(RedisSinkDeploymentType.CLUSTER); + when(redisSinkConfig.getSinkRedisUrls()).thenReturn("0.0.0.0:0, 1.1.1.1:1"); + + RedisClient client = RedisClientFactory.getClient(redisSinkConfig, statsDReporter); + + Assert.assertEquals(RedisClusterClient.class, client.getClass()); + } + + @Test + public void shouldGetClusterClientWhenURLHasSpaces() { + when(redisSinkConfig.getSinkRedisTtlType()).thenReturn(RedisSinkTtlType.DURATION); + when(redisSinkConfig.getSinkRedisDeploymentType()).thenReturn(RedisSinkDeploymentType.CLUSTER); + when(redisSinkConfig.getSinkRedisUrls()).thenReturn(" 0.0.0.0:0, 1.1.1.1:1 "); + + RedisClient client = RedisClientFactory.getClient(redisSinkConfig, statsDReporter); + + Assert.assertEquals(RedisClusterClient.class, client.getClass()); + } + + @Test + public void shouldThrowExceptionWhenUrlIsInvalidForCluster() { + expectedException.expect(ConfigurationException.class); + expectedException.expectMessage("Invalid url(s) for redis cluster: localhost:6379,localhost:6378,localhost"); + + when(redisSinkConfig.getSinkRedisTtlType()).thenReturn(RedisSinkTtlType.DURATION); + when(redisSinkConfig.getSinkRedisDeploymentType()).thenReturn(RedisSinkDeploymentType.CLUSTER); + when(redisSinkConfig.getSinkRedisUrls()).thenReturn("localhost:6379,localhost:6378,localhost"); + + RedisClient client = RedisClientFactory.getClient(redisSinkConfig, statsDReporter); + } + + @Test + public void shouldThrowExceptionWhenUrlIsInvalidForStandalone() { + expectedException.expect(ConfigurationException.class); + expectedException.expectMessage("Invalid url for redis standalone: localhost"); + + when(redisSinkConfig.getSinkRedisTtlType()).thenReturn(RedisSinkTtlType.DURATION); + when(redisSinkConfig.getSinkRedisDeploymentType()).thenReturn(RedisSinkDeploymentType.STANDALONE); + when(redisSinkConfig.getSinkRedisUrls()).thenReturn("localhost"); + + RedisClientFactory.getClient(redisSinkConfig, statsDReporter); + } +} diff --git a/src/test/java/io/odpf/depot/redis/client/RedisClusterClientTest.java b/src/test/java/io/odpf/depot/redis/client/RedisClusterClientTest.java new file mode 100644 index 00000000..db051d90 --- /dev/null +++ b/src/test/java/io/odpf/depot/redis/client/RedisClusterClientTest.java @@ -0,0 +1,64 @@ +package io.odpf.depot.redis.client; + +import io.odpf.depot.metrics.Instrumentation; +import io.odpf.depot.redis.client.response.RedisClusterResponse; +import io.odpf.depot.redis.client.response.RedisResponse; +import io.odpf.depot.redis.record.RedisRecord; +import io.odpf.depot.redis.ttl.RedisTtl; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import redis.clients.jedis.JedisCluster; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.IntStream; + +@RunWith(MockitoJUnitRunner.class) +public class RedisClusterClientTest { + @Mock + private Instrumentation instrumentation; + @Mock + private RedisTtl redisTTL; + @Mock + private JedisCluster jedisCluster; + + @Test + public void shouldSendToRedisCluster() { + RedisClient redisClient = new RedisClusterClient(instrumentation, redisTTL, jedisCluster); + List redisRecords = new ArrayList() {{ + add(Mockito.mock(RedisRecord.class)); + add(Mockito.mock(RedisRecord.class)); + add(Mockito.mock(RedisRecord.class)); + add(Mockito.mock(RedisRecord.class)); + add(Mockito.mock(RedisRecord.class)); + add(Mockito.mock(RedisRecord.class)); + }}; + List responses = new ArrayList() {{ + add(Mockito.mock(RedisClusterResponse.class)); + add(Mockito.mock(RedisClusterResponse.class)); + add(Mockito.mock(RedisClusterResponse.class)); + add(Mockito.mock(RedisClusterResponse.class)); + add(Mockito.mock(RedisClusterResponse.class)); + add(Mockito.mock(RedisClusterResponse.class)); + }}; + IntStream.range(0, redisRecords.size()).forEach( + index -> Mockito.when(redisRecords.get(index).send(jedisCluster, redisTTL)).thenReturn(responses.get(index)) + ); + List actualResponse = redisClient.send(redisRecords); + IntStream.range(0, redisRecords.size()).forEach( + index -> Assert.assertEquals(responses.get(index), actualResponse.get(index))); + } + + @Test + public void shouldCallClose() throws IOException { + RedisClient redisClient = new RedisClusterClient(instrumentation, redisTTL, jedisCluster); + redisClient.close(); + Mockito.verify(instrumentation, Mockito.times(1)).logInfo("Closing Jedis client"); + Mockito.verify(jedisCluster, Mockito.times(1)).close(); + } +} diff --git a/src/test/java/io/odpf/depot/redis/client/RedisStandaloneClientTest.java b/src/test/java/io/odpf/depot/redis/client/RedisStandaloneClientTest.java new file mode 100644 index 00000000..3d5ed1a8 --- /dev/null +++ b/src/test/java/io/odpf/depot/redis/client/RedisStandaloneClientTest.java @@ -0,0 +1,83 @@ +package io.odpf.depot.redis.client; + +import io.odpf.depot.metrics.Instrumentation; +import io.odpf.depot.redis.client.response.RedisResponse; +import io.odpf.depot.redis.client.response.RedisStandaloneResponse; +import io.odpf.depot.redis.record.RedisRecord; +import io.odpf.depot.redis.ttl.RedisTtl; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.Pipeline; +import redis.clients.jedis.Response; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.IntStream; + + +@RunWith(MockitoJUnitRunner.class) +public class RedisStandaloneClientTest { + @Mock + private Instrumentation instrumentation; + @Mock + private RedisTtl redisTTL; + @Mock + private Jedis jedis; + + @Test + public void shouldCloseTheClient() throws IOException { + RedisClient redisClient = new RedisStandaloneClient(instrumentation, redisTTL, jedis); + redisClient.close(); + + Mockito.verify(instrumentation, Mockito.times(1)).logInfo("Closing Jedis client"); + Mockito.verify(jedis, Mockito.times(1)).close(); + } + + @Test + public void shouldSendRecordsToJedis() { + RedisClient redisClient = new RedisStandaloneClient(instrumentation, redisTTL, jedis); + Pipeline pipeline = Mockito.mock(Pipeline.class); + Response response = Mockito.mock(Response.class); + Mockito.when(jedis.pipelined()).thenReturn(pipeline); + Mockito.when(pipeline.exec()).thenReturn(response); + Object ob = new Object(); + Mockito.when(response.get()).thenReturn(ob); + List redisRecords = new ArrayList() {{ + add(Mockito.mock(RedisRecord.class)); + add(Mockito.mock(RedisRecord.class)); + add(Mockito.mock(RedisRecord.class)); + add(Mockito.mock(RedisRecord.class)); + add(Mockito.mock(RedisRecord.class)); + add(Mockito.mock(RedisRecord.class)); + }}; + List responses = new ArrayList() {{ + add(Mockito.mock(RedisStandaloneResponse.class)); + add(Mockito.mock(RedisStandaloneResponse.class)); + add(Mockito.mock(RedisStandaloneResponse.class)); + add(Mockito.mock(RedisStandaloneResponse.class)); + add(Mockito.mock(RedisStandaloneResponse.class)); + add(Mockito.mock(RedisStandaloneResponse.class)); + }}; + IntStream.range(0, redisRecords.size()).forEach( + index -> { + Mockito.when(redisRecords.get(index).send(pipeline, redisTTL)).thenReturn(responses.get(index)); + Mockito.when(responses.get(index).process()).thenReturn(responses.get(index)); + } + ); + List actualResponses = redisClient.send(redisRecords); + Mockito.verify(pipeline, Mockito.times(1)).multi(); + Mockito.verify(pipeline, Mockito.times(1)).sync(); + Mockito.verify(instrumentation, Mockito.times(1)).logDebug("jedis responses: {}", ob); + IntStream.range(0, actualResponses.size()).forEach( + index -> { + Assert.assertEquals(responses.get(index), actualResponses.get(index)); + } + ); + } +} diff --git a/src/test/java/io/odpf/depot/redis/client/entry/RedisHashSetFieldEntryTest.java b/src/test/java/io/odpf/depot/redis/client/entry/RedisHashSetFieldEntryTest.java new file mode 100644 index 00000000..574d793b --- /dev/null +++ b/src/test/java/io/odpf/depot/redis/client/entry/RedisHashSetFieldEntryTest.java @@ -0,0 +1,156 @@ +package io.odpf.depot.redis.client.entry; + +import io.odpf.depot.metrics.Instrumentation; +import io.odpf.depot.redis.client.response.RedisClusterResponse; +import io.odpf.depot.redis.client.response.RedisStandaloneResponse; +import io.odpf.depot.redis.ttl.DurationTtl; +import io.odpf.depot.redis.ttl.NoRedisTtl; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import redis.clients.jedis.JedisCluster; +import redis.clients.jedis.Pipeline; +import redis.clients.jedis.Response; +import redis.clients.jedis.exceptions.JedisException; + +import static org.mockito.Mockito.*; + +@RunWith(MockitoJUnitRunner.class) +public class RedisHashSetFieldEntryTest { + @Mock + private Instrumentation instrumentation; + @Mock + private Pipeline pipeline; + @Mock + private JedisCluster jedisCluster; + private RedisHashSetFieldEntry redisHashSetFieldEntry; + + @Before + public void setup() { + redisHashSetFieldEntry = new RedisHashSetFieldEntry("test-key", "test-field", "test-value", instrumentation); + } + + @Test + public void shouldSentToRedisForCluster() { + when(jedisCluster.hset("test-key", "test-field", "test-value")).thenReturn(9L); + RedisClusterResponse clusterResponse = redisHashSetFieldEntry.send(jedisCluster, new NoRedisTtl()); + Assert.assertFalse(clusterResponse.isFailed()); + verify(instrumentation, times(1)).logDebug("key: {}, field: {}, value: {}", "test-key", "test-field", "test-value"); + Assert.assertEquals("HSET: 9, TTL: NoOp", clusterResponse.getMessage()); + } + + @Test + public void shouldSentToRedisForClusterWithTTL() { + when(jedisCluster.hset("test-key", "test-field", "test-value")).thenReturn(9L); + when(jedisCluster.expire("test-key", 1000)).thenReturn(1L); + RedisClusterResponse clusterResponse = redisHashSetFieldEntry.send(jedisCluster, new DurationTtl(1000)); + Assert.assertFalse(clusterResponse.isFailed()); + verify(instrumentation, times(1)).logDebug("key: {}, field: {}, value: {}", "test-key", "test-field", "test-value"); + Assert.assertEquals("HSET: 9, TTL: UPDATED", clusterResponse.getMessage()); + } + + @Test + public void shouldSentToRedisForClusterWithTTLNotUpdated() { + when(jedisCluster.hset("test-key", "test-field", "test-value")).thenReturn(9L); + when(jedisCluster.expire("test-key", 1000)).thenReturn(0L); + RedisClusterResponse clusterResponse = redisHashSetFieldEntry.send(jedisCluster, new DurationTtl(1000)); + Assert.assertFalse(clusterResponse.isFailed()); + verify(instrumentation, times(1)).logDebug("key: {}, field: {}, value: {}", "test-key", "test-field", "test-value"); + Assert.assertEquals("HSET: 9, TTL: NOT UPDATED", clusterResponse.getMessage()); + } + + @Test + public void shouldReportFailedForJedisExceptionForCluster() { + when(jedisCluster.hset("test-key", "test-field", "test-value")).thenThrow(new JedisException("jedis error occurred")); + RedisClusterResponse clusterResponse = redisHashSetFieldEntry.send(jedisCluster, new NoRedisTtl()); + Assert.assertTrue(clusterResponse.isFailed()); + Assert.assertEquals("jedis error occurred", clusterResponse.getMessage()); + } + + @Test + public void shouldReportFailedForJedisExceptionFromTTLForCluster() { + when(jedisCluster.hset("test-key", "test-field", "test-value")).thenReturn(10L); + when(jedisCluster.expire("test-key", 1000)).thenThrow(new JedisException("jedis error occurred")); + RedisClusterResponse clusterResponse = redisHashSetFieldEntry.send(jedisCluster, new DurationTtl(1000)); + Assert.assertTrue(clusterResponse.isFailed()); + Assert.assertEquals("jedis error occurred", clusterResponse.getMessage()); + } + + @Test + public void shouldGetSetEntryToString() { + String expected = "RedisHashSetFieldEntry Key test-key, Field test-field, Value test-value"; + Assert.assertEquals(expected, redisHashSetFieldEntry.toString()); + } + + + @Test + public void shouldSentToRedisForStandAlone() { + Response r = Mockito.mock(Response.class); + when(r.get()).thenReturn(9L); + when(pipeline.hset("test-key", "test-field", "test-value")).thenReturn(r); + RedisStandaloneResponse standaloneResponse = redisHashSetFieldEntry.send(pipeline, new NoRedisTtl()); + standaloneResponse.process(); + Assert.assertFalse(standaloneResponse.isFailed()); + verify(instrumentation, times(1)).logDebug("key: {}, field: {}, value: {}", "test-key", "test-field", "test-value"); + Assert.assertEquals("HSET: 9, TTL: NoOp", standaloneResponse.getMessage()); + } + + @Test + public void shouldSentToRedisForStandaloneWithTTL() { + Response r = Mockito.mock(Response.class); + when(r.get()).thenReturn(9L); + Response tr = Mockito.mock(Response.class); + when(tr.get()).thenReturn(1L); + when(pipeline.hset("test-key", "test-field", "test-value")).thenReturn(r); + when(pipeline.expire("test-key", 1000)).thenReturn(tr); + RedisStandaloneResponse standaloneResponse = redisHashSetFieldEntry.send(pipeline, new DurationTtl(1000)); + standaloneResponse.process(); + Assert.assertFalse(standaloneResponse.isFailed()); + verify(instrumentation, times(1)).logDebug("key: {}, field: {}, value: {}", "test-key", "test-field", "test-value"); + Assert.assertEquals("HSET: 9, TTL: UPDATED", standaloneResponse.getMessage()); + } + + @Test + public void shouldSentToRedisForStandaloneWithTTLNotUpdated() { + Response r = Mockito.mock(Response.class); + when(r.get()).thenReturn(9L); + Response tr = Mockito.mock(Response.class); + when(tr.get()).thenReturn(0L); + when(pipeline.hset("test-key", "test-field", "test-value")).thenReturn(r); + when(pipeline.expire("test-key", 1000)).thenReturn(tr); + RedisStandaloneResponse standaloneResponse = redisHashSetFieldEntry.send(pipeline, new DurationTtl(1000)); + standaloneResponse.process(); + Assert.assertFalse(standaloneResponse.isFailed()); + verify(instrumentation, times(1)).logDebug("key: {}, field: {}, value: {}", "test-key", "test-field", "test-value"); + Assert.assertEquals("HSET: 9, TTL: NOT UPDATED", standaloneResponse.getMessage()); + } + + @Test + public void shouldReportFailedForJedisExceptionForStandalone() { + Response r = Mockito.mock(Response.class); + when(pipeline.hset("test-key", "test-field", "test-value")).thenReturn(r); + when(r.get()).thenThrow(new JedisException("jedis error occurred")); + RedisStandaloneResponse standaloneResponse = redisHashSetFieldEntry.send(pipeline, new NoRedisTtl()); + standaloneResponse.process(); + Assert.assertTrue(standaloneResponse.isFailed()); + Assert.assertEquals("jedis error occurred", standaloneResponse.getMessage()); + } + + @Test + public void shouldReportFailedForJedisExceptionFromTTLForStandalone() { + Response r = Mockito.mock(Response.class); + when(r.get()).thenReturn(9L); + Response tr = Mockito.mock(Response.class); + when(tr.get()).thenThrow(new JedisException("jedis error occurred")); + when(pipeline.hset("test-key", "test-field", "test-value")).thenReturn(r); + when(pipeline.expire("test-key", 1000)).thenReturn(tr); + RedisStandaloneResponse standaloneResponse = redisHashSetFieldEntry.send(pipeline, new DurationTtl(1000)); + standaloneResponse.process(); + Assert.assertTrue(standaloneResponse.isFailed()); + Assert.assertEquals("jedis error occurred", standaloneResponse.getMessage()); + } +} diff --git a/src/test/java/io/odpf/depot/redis/client/entry/RedisKeyValueEntryTest.java b/src/test/java/io/odpf/depot/redis/client/entry/RedisKeyValueEntryTest.java new file mode 100644 index 00000000..df928442 --- /dev/null +++ b/src/test/java/io/odpf/depot/redis/client/entry/RedisKeyValueEntryTest.java @@ -0,0 +1,158 @@ +package io.odpf.depot.redis.client.entry; + + +import io.odpf.depot.metrics.Instrumentation; +import io.odpf.depot.redis.client.response.RedisClusterResponse; +import io.odpf.depot.redis.client.response.RedisStandaloneResponse; +import io.odpf.depot.redis.ttl.DurationTtl; +import io.odpf.depot.redis.ttl.NoRedisTtl; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import redis.clients.jedis.JedisCluster; +import redis.clients.jedis.Pipeline; +import redis.clients.jedis.Response; +import redis.clients.jedis.exceptions.JedisException; + +import static org.mockito.Mockito.*; + +@RunWith(MockitoJUnitRunner.class) +public class RedisKeyValueEntryTest { + @Mock + private Instrumentation instrumentation; + @Mock + private Pipeline pipeline; + @Mock + private JedisCluster jedisCluster; + private RedisKeyValueEntry redisKeyValueEntry; + + @Before + public void setup() { + redisKeyValueEntry = new RedisKeyValueEntry("test-key", "test-value", instrumentation); + } + + @Test + public void shouldSentToRedisForCluster() { + when(jedisCluster.set("test-key", "test-value")).thenReturn("OK"); + RedisClusterResponse clusterResponse = redisKeyValueEntry.send(jedisCluster, new NoRedisTtl()); + Assert.assertFalse(clusterResponse.isFailed()); + verify(instrumentation, times(1)).logDebug("key: {}, value: {}", "test-key", "test-value"); + Assert.assertEquals("SET: OK, TTL: NoOp", clusterResponse.getMessage()); + } + + @Test + public void shouldSentToRedisForClusterWithTTL() { + when(jedisCluster.set("test-key", "test-value")).thenReturn("OK"); + when(jedisCluster.expire("test-key", 1000)).thenReturn(1L); + RedisClusterResponse clusterResponse = redisKeyValueEntry.send(jedisCluster, new DurationTtl(1000)); + Assert.assertFalse(clusterResponse.isFailed()); + verify(instrumentation, times(1)).logDebug("key: {}, value: {}", "test-key", "test-value"); + Assert.assertEquals("SET: OK, TTL: UPDATED", clusterResponse.getMessage()); + } + + @Test + public void shouldSentToRedisForClusterWithTTLNotUpdated() { + when(jedisCluster.set("test-key", "test-value")).thenReturn("OK"); + when(jedisCluster.expire("test-key", 1000)).thenReturn(0L); + RedisClusterResponse clusterResponse = redisKeyValueEntry.send(jedisCluster, new DurationTtl(1000)); + Assert.assertFalse(clusterResponse.isFailed()); + verify(instrumentation, times(1)).logDebug("key: {}, value: {}", "test-key", "test-value"); + Assert.assertEquals("SET: OK, TTL: NOT UPDATED", clusterResponse.getMessage()); + } + + @Test + public void shouldReportFailedForJedisExceptionForCluster() { + when(jedisCluster.set("test-key", "test-value")).thenThrow(new JedisException("jedis error occurred")); + RedisClusterResponse clusterResponse = redisKeyValueEntry.send(jedisCluster, new NoRedisTtl()); + Assert.assertTrue(clusterResponse.isFailed()); + Assert.assertEquals("jedis error occurred", clusterResponse.getMessage()); + } + + @Test + public void shouldReportFailedForJedisExceptionFromTTLForCluster() { + when(jedisCluster.set("test-key", "test-value")).thenReturn("OK"); + when(jedisCluster.expire("test-key", 1000)).thenThrow(new JedisException("jedis error occurred")); + RedisClusterResponse clusterResponse = redisKeyValueEntry.send(jedisCluster, new DurationTtl(1000)); + Assert.assertTrue(clusterResponse.isFailed()); + Assert.assertEquals("jedis error occurred", clusterResponse.getMessage()); + } + + @Test + public void shouldGetEntryToString() { + String expected = "RedisKeyValueEntry: Key test-key, Value test-value"; + Assert.assertEquals(expected, redisKeyValueEntry.toString()); + } + + + @Test + public void shouldSentToRedisForStandAlone() { + Response r = Mockito.mock(Response.class); + when(r.get()).thenReturn("OK"); + when(pipeline.set("test-key", "test-value")).thenReturn(r); + RedisStandaloneResponse standaloneResponse = redisKeyValueEntry.send(pipeline, new NoRedisTtl()); + standaloneResponse.process(); + Assert.assertFalse(standaloneResponse.isFailed()); + verify(instrumentation, times(1)).logDebug("key: {}, value: {}", "test-key", "test-value"); + Assert.assertEquals("SET: OK, TTL: NoOp", standaloneResponse.getMessage()); + } + + @Test + public void shouldSentToRedisForStandaloneWithTTL() { + Response r = Mockito.mock(Response.class); + when(r.get()).thenReturn("OK"); + Response tr = Mockito.mock(Response.class); + when(tr.get()).thenReturn(1L); + when(pipeline.set("test-key", "test-value")).thenReturn(r); + when(pipeline.expire("test-key", 1000)).thenReturn(tr); + RedisStandaloneResponse standaloneResponse = redisKeyValueEntry.send(pipeline, new DurationTtl(1000)); + standaloneResponse.process(); + Assert.assertFalse(standaloneResponse.isFailed()); + verify(instrumentation, times(1)).logDebug("key: {}, value: {}", "test-key", "test-value"); + Assert.assertEquals("SET: OK, TTL: UPDATED", standaloneResponse.getMessage()); + } + + @Test + public void shouldSentToRedisForStandaloneWithTTLNotUpdated() { + Response r = Mockito.mock(Response.class); + when(r.get()).thenReturn("OK"); + Response tr = Mockito.mock(Response.class); + when(tr.get()).thenReturn(0L); + when(pipeline.set("test-key", "test-value")).thenReturn(r); + when(pipeline.expire("test-key", 1000)).thenReturn(tr); + RedisStandaloneResponse standaloneResponse = redisKeyValueEntry.send(pipeline, new DurationTtl(1000)); + standaloneResponse.process(); + Assert.assertFalse(standaloneResponse.isFailed()); + verify(instrumentation, times(1)).logDebug("key: {}, value: {}", "test-key", "test-value"); + Assert.assertEquals("SET: OK, TTL: NOT UPDATED", standaloneResponse.getMessage()); + } + + @Test + public void shouldReportFailedForJedisExceptionForStandalone() { + Response r = Mockito.mock(Response.class); + when(pipeline.set("test-key", "test-value")).thenReturn(r); + when(r.get()).thenThrow(new JedisException("jedis error occurred")); + RedisStandaloneResponse standaloneResponse = redisKeyValueEntry.send(pipeline, new NoRedisTtl()); + standaloneResponse.process(); + Assert.assertTrue(standaloneResponse.isFailed()); + Assert.assertEquals("jedis error occurred", standaloneResponse.getMessage()); + } + + @Test + public void shouldReportFailedForJedisExceptionFromTTLForStandalone() { + Response r = Mockito.mock(Response.class); + when(r.get()).thenReturn("OK"); + Response tr = Mockito.mock(Response.class); + when(tr.get()).thenThrow(new JedisException("jedis error occurred")); + when(pipeline.set("test-key", "test-value")).thenReturn(r); + when(pipeline.expire("test-key", 1000)).thenReturn(tr); + RedisStandaloneResponse standaloneResponse = redisKeyValueEntry.send(pipeline, new DurationTtl(1000)); + standaloneResponse.process(); + Assert.assertTrue(standaloneResponse.isFailed()); + Assert.assertEquals("jedis error occurred", standaloneResponse.getMessage()); + } +} + diff --git a/src/test/java/io/odpf/depot/redis/client/entry/RedisListEntryTest.java b/src/test/java/io/odpf/depot/redis/client/entry/RedisListEntryTest.java new file mode 100644 index 00000000..8a69a43e --- /dev/null +++ b/src/test/java/io/odpf/depot/redis/client/entry/RedisListEntryTest.java @@ -0,0 +1,156 @@ +package io.odpf.depot.redis.client.entry; + +import io.odpf.depot.metrics.Instrumentation; +import io.odpf.depot.redis.client.response.RedisClusterResponse; +import io.odpf.depot.redis.client.response.RedisStandaloneResponse; +import io.odpf.depot.redis.ttl.DurationTtl; +import io.odpf.depot.redis.ttl.NoRedisTtl; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import redis.clients.jedis.JedisCluster; +import redis.clients.jedis.Pipeline; +import redis.clients.jedis.Response; +import redis.clients.jedis.exceptions.JedisException; + +import static org.mockito.Mockito.*; + +@RunWith(MockitoJUnitRunner.class) +public class RedisListEntryTest { + @Mock + private Instrumentation instrumentation; + @Mock + private Pipeline pipeline; + @Mock + private JedisCluster jedisCluster; + private RedisListEntry redisListEntry; + + @Before + public void setup() { + redisListEntry = new RedisListEntry("test-key", "test-value", instrumentation); + } + + @Test + public void shouldSentToRedisForCluster() { + when(jedisCluster.lpush("test-key", "test-value")).thenReturn(9L); + RedisClusterResponse clusterResponse = redisListEntry.send(jedisCluster, new NoRedisTtl()); + Assert.assertFalse(clusterResponse.isFailed()); + verify(instrumentation, times(1)).logDebug("key: {}, value: {}", "test-key", "test-value"); + Assert.assertEquals("LPUSH: 9, TTL: NoOp", clusterResponse.getMessage()); + } + + @Test + public void shouldSentToRedisForClusterWithTTL() { + when(jedisCluster.lpush("test-key", "test-value")).thenReturn(9L); + when(jedisCluster.expire("test-key", 1000)).thenReturn(1L); + RedisClusterResponse clusterResponse = redisListEntry.send(jedisCluster, new DurationTtl(1000)); + Assert.assertFalse(clusterResponse.isFailed()); + verify(instrumentation, times(1)).logDebug("key: {}, value: {}", "test-key", "test-value"); + Assert.assertEquals("LPUSH: 9, TTL: UPDATED", clusterResponse.getMessage()); + } + + @Test + public void shouldSentToRedisForClusterWithTTLNotUpdated() { + when(jedisCluster.lpush("test-key", "test-value")).thenReturn(9L); + when(jedisCluster.expire("test-key", 1000)).thenReturn(0L); + RedisClusterResponse clusterResponse = redisListEntry.send(jedisCluster, new DurationTtl(1000)); + Assert.assertFalse(clusterResponse.isFailed()); + verify(instrumentation, times(1)).logDebug("key: {}, value: {}", "test-key", "test-value"); + Assert.assertEquals("LPUSH: 9, TTL: NOT UPDATED", clusterResponse.getMessage()); + } + + @Test + public void shouldReportFailedForJedisExceptionForCluster() { + when(jedisCluster.lpush("test-key", "test-value")).thenThrow(new JedisException("jedis error occurred")); + RedisClusterResponse clusterResponse = redisListEntry.send(jedisCluster, new NoRedisTtl()); + Assert.assertTrue(clusterResponse.isFailed()); + Assert.assertEquals("jedis error occurred", clusterResponse.getMessage()); + } + + @Test + public void shouldReportFailedForJedisExceptionFromTTLForCluster() { + when(jedisCluster.lpush("test-key", "test-value")).thenReturn(9L); + when(jedisCluster.expire("test-key", 1000)).thenThrow(new JedisException("jedis error occurred")); + RedisClusterResponse clusterResponse = redisListEntry.send(jedisCluster, new DurationTtl(1000)); + Assert.assertTrue(clusterResponse.isFailed()); + Assert.assertEquals("jedis error occurred", clusterResponse.getMessage()); + } + + @Test + public void shouldGetEntryToString() { + String expected = "RedisListEntry: Key test-key, Value test-value"; + Assert.assertEquals(expected, redisListEntry.toString()); + } + + + @Test + public void shouldSentToRedisForStandAlone() { + Response r = Mockito.mock(Response.class); + when(r.get()).thenReturn(9L); + when(pipeline.lpush("test-key", "test-value")).thenReturn(r); + RedisStandaloneResponse standaloneResponse = redisListEntry.send(pipeline, new NoRedisTtl()); + standaloneResponse.process(); + Assert.assertFalse(standaloneResponse.isFailed()); + verify(instrumentation, times(1)).logDebug("key: {}, value: {}", "test-key", "test-value"); + Assert.assertEquals("LPUSH: 9, TTL: NoOp", standaloneResponse.getMessage()); + } + + @Test + public void shouldSentToRedisForStandaloneWithTTL() { + Response r = Mockito.mock(Response.class); + when(r.get()).thenReturn(9L); + Response tr = Mockito.mock(Response.class); + when(tr.get()).thenReturn(1L); + when(pipeline.lpush("test-key", "test-value")).thenReturn(r); + when(pipeline.expire("test-key", 1000)).thenReturn(tr); + RedisStandaloneResponse standaloneResponse = redisListEntry.send(pipeline, new DurationTtl(1000)); + standaloneResponse.process(); + Assert.assertFalse(standaloneResponse.isFailed()); + verify(instrumentation, times(1)).logDebug("key: {}, value: {}", "test-key", "test-value"); + Assert.assertEquals("LPUSH: 9, TTL: UPDATED", standaloneResponse.getMessage()); + } + + @Test + public void shouldSentToRedisForStandaloneWithTTLNotUpdated() { + Response r = Mockito.mock(Response.class); + when(r.get()).thenReturn(9L); + Response tr = Mockito.mock(Response.class); + when(tr.get()).thenReturn(0L); + when(pipeline.lpush("test-key", "test-value")).thenReturn(r); + when(pipeline.expire("test-key", 1000)).thenReturn(tr); + RedisStandaloneResponse standaloneResponse = redisListEntry.send(pipeline, new DurationTtl(1000)); + standaloneResponse.process(); + Assert.assertFalse(standaloneResponse.isFailed()); + verify(instrumentation, times(1)).logDebug("key: {}, value: {}", "test-key", "test-value"); + Assert.assertEquals("LPUSH: 9, TTL: NOT UPDATED", standaloneResponse.getMessage()); + } + + @Test + public void shouldReportFailedForJedisExceptionForStandalone() { + Response r = Mockito.mock(Response.class); + when(pipeline.lpush("test-key", "test-value")).thenReturn(r); + when(r.get()).thenThrow(new JedisException("jedis error occurred")); + RedisStandaloneResponse standaloneResponse = redisListEntry.send(pipeline, new NoRedisTtl()); + standaloneResponse.process(); + Assert.assertTrue(standaloneResponse.isFailed()); + Assert.assertEquals("jedis error occurred", standaloneResponse.getMessage()); + } + + @Test + public void shouldReportFailedForJedisExceptionFromTTLForStandalone() { + Response r = Mockito.mock(Response.class); + when(r.get()).thenReturn(9L); + Response tr = Mockito.mock(Response.class); + when(tr.get()).thenThrow(new JedisException("jedis error occurred")); + when(pipeline.lpush("test-key", "test-value")).thenReturn(r); + when(pipeline.expire("test-key", 1000)).thenReturn(tr); + RedisStandaloneResponse standaloneResponse = redisListEntry.send(pipeline, new DurationTtl(1000)); + standaloneResponse.process(); + Assert.assertTrue(standaloneResponse.isFailed()); + Assert.assertEquals("jedis error occurred", standaloneResponse.getMessage()); + } +} diff --git a/src/test/java/io/odpf/depot/redis/client/response/RedisClusterResponseTest.java b/src/test/java/io/odpf/depot/redis/client/response/RedisClusterResponseTest.java new file mode 100644 index 00000000..0f8709f9 --- /dev/null +++ b/src/test/java/io/odpf/depot/redis/client/response/RedisClusterResponseTest.java @@ -0,0 +1,24 @@ +package io.odpf.depot.redis.client.response; + +import org.junit.Assert; +import org.junit.Test; + +public class RedisClusterResponseTest { + private RedisClusterResponse redisClusterResponse; + + @Test + public void shouldReportWhenSuccess() { + String response = "Success"; + Long ttlResponse = 1L; + redisClusterResponse = new RedisClusterResponse("SET", response, ttlResponse); + Assert.assertFalse(redisClusterResponse.isFailed()); + Assert.assertEquals("SET: Success, TTL: UPDATED", redisClusterResponse.getMessage()); + } + + @Test + public void shouldReportWhenFailed() { + redisClusterResponse = new RedisClusterResponse("Failed"); + Assert.assertTrue(redisClusterResponse.isFailed()); + Assert.assertEquals("Failed", redisClusterResponse.getMessage()); + } +} diff --git a/src/test/java/io/odpf/depot/redis/client/response/RedisStandaloneResponseTest.java b/src/test/java/io/odpf/depot/redis/client/response/RedisStandaloneResponseTest.java new file mode 100644 index 00000000..24ced647 --- /dev/null +++ b/src/test/java/io/odpf/depot/redis/client/response/RedisStandaloneResponseTest.java @@ -0,0 +1,37 @@ +package io.odpf.depot.redis.client.response; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; +import redis.clients.jedis.Response; +import redis.clients.jedis.exceptions.JedisException; + +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class RedisStandaloneResponseTest { + @Mock + private Response response; + @Mock + private Response ttlResponse; + private RedisStandaloneResponse redisResponse; + + @Test + public void shouldReportNotFailedWhenJedisExceptionNotThrown() { + when(response.get()).thenReturn("Success response"); + when(ttlResponse.get()).thenReturn(1L); + redisResponse = new RedisStandaloneResponse("SET", response, ttlResponse); + Assert.assertFalse(redisResponse.process().isFailed()); + Assert.assertEquals("SET: Success response, TTL: UPDATED", redisResponse.process().getMessage()); + } + + @Test + public void shouldReportFailedWhenJedisExceptionThrown() { + when(response.get()).thenThrow(new JedisException("Failed response")); + redisResponse = new RedisStandaloneResponse("SET", response, ttlResponse); + Assert.assertTrue(redisResponse.process().isFailed()); + Assert.assertEquals("Failed response", redisResponse.process().getMessage()); + } +} diff --git a/src/test/java/io/odpf/depot/redis/parsers/RedisEntryParserFactoryTest.java b/src/test/java/io/odpf/depot/redis/parsers/RedisEntryParserFactoryTest.java new file mode 100644 index 00000000..73e7b44a --- /dev/null +++ b/src/test/java/io/odpf/depot/redis/parsers/RedisEntryParserFactoryTest.java @@ -0,0 +1,109 @@ +package io.odpf.depot.redis.parsers; + +import io.odpf.depot.config.RedisSinkConfig; +import io.odpf.depot.config.converter.JsonToPropertiesConverter; +import io.odpf.depot.message.OdpfMessageSchema; +import io.odpf.depot.metrics.StatsDReporter; +import io.odpf.depot.redis.enums.RedisSinkDataType; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class RedisEntryParserFactoryTest { + @Mock + private RedisSinkConfig redisSinkConfig; + @Mock + private StatsDReporter statsDReporter; + @Mock + private OdpfMessageSchema schema; + + @Before + public void setup() { + when(redisSinkConfig.getSinkRedisKeyTemplate()).thenReturn("redis-key"); + when(redisSinkConfig.getSinkRedisKeyValueDataFieldName()).thenReturn("keyvalue-field"); + when(redisSinkConfig.getSinkRedisListDataFieldName()).thenReturn("list-field"); + when(redisSinkConfig.getSinkRedisHashsetFieldToColumnMapping()).thenReturn(new JsonToPropertiesConverter().convert(null, "{\"field\":\"column\"}")); + } + + @Test + public void shouldReturnNewRedisListParser() { + when(redisSinkConfig.getSinkRedisDataType()).thenReturn(RedisSinkDataType.LIST); + RedisEntryParser parser = RedisEntryParserFactory.getRedisEntryParser(redisSinkConfig, statsDReporter, schema); + assertEquals(RedisListEntryParser.class, parser.getClass()); + } + + @Test + public void shouldReturnNewRedisHashSetParser() { + when(redisSinkConfig.getSinkRedisDataType()).thenReturn(RedisSinkDataType.HASHSET); + RedisEntryParser parser = RedisEntryParserFactory.getRedisEntryParser(redisSinkConfig, statsDReporter, schema); + assertEquals(RedisHashSetEntryParser.class, parser.getClass()); + } + + @Test + public void shouldReturnNewRedisKeyValueParser() { + when(redisSinkConfig.getSinkRedisDataType()).thenReturn(RedisSinkDataType.KEYVALUE); + RedisEntryParser parser = RedisEntryParserFactory.getRedisEntryParser(redisSinkConfig, statsDReporter, schema); + assertEquals(RedisKeyValueEntryParser.class, parser.getClass()); + } + + @Test + public void shouldThrowExceptionForEmptyMappingForHashSet() { + when(redisSinkConfig.getSinkRedisDataType()).thenReturn(RedisSinkDataType.HASHSET); + when(redisSinkConfig.getSinkRedisHashsetFieldToColumnMapping()).thenReturn(new JsonToPropertiesConverter().convert(null, "")); + IllegalArgumentException e = Assert.assertThrows(IllegalArgumentException.class, + () -> RedisEntryParserFactory.getRedisEntryParser(redisSinkConfig, statsDReporter, schema)); + assertEquals("Empty config SINK_REDIS_HASHSET_FIELD_TO_COLUMN_MAPPING found", e.getMessage()); + } + + @Test + public void shouldThrowExceptionForNullMappingForHashSet() { + when(redisSinkConfig.getSinkRedisDataType()).thenReturn(RedisSinkDataType.HASHSET); + when(redisSinkConfig.getSinkRedisHashsetFieldToColumnMapping()).thenReturn(new JsonToPropertiesConverter().convert(null, null)); + IllegalArgumentException e = Assert.assertThrows(IllegalArgumentException.class, + () -> RedisEntryParserFactory.getRedisEntryParser(redisSinkConfig, statsDReporter, schema)); + assertEquals("Empty config SINK_REDIS_HASHSET_FIELD_TO_COLUMN_MAPPING found", e.getMessage()); + } + + @Test + public void shouldThrowExceptionForEmptyMappingKeyHashSet() { + when(redisSinkConfig.getSinkRedisDataType()).thenReturn(RedisSinkDataType.HASHSET); + when(redisSinkConfig.getSinkRedisHashsetFieldToColumnMapping()).thenReturn(new JsonToPropertiesConverter().convert(null, "{\"order_details\":\"\"}")); + IllegalArgumentException e = Assert.assertThrows(IllegalArgumentException.class, + () -> RedisEntryParserFactory.getRedisEntryParser(redisSinkConfig, statsDReporter, schema)); + assertEquals("Template '' is invalid", e.getMessage()); + } + + @Test + public void shouldThrowExceptionForEmptyKeyValueDataFieldName() { + when(redisSinkConfig.getSinkRedisDataType()).thenReturn(RedisSinkDataType.KEYVALUE); + when(redisSinkConfig.getSinkRedisKeyValueDataFieldName()).thenReturn(""); + IllegalArgumentException illegalArgumentException = + assertThrows(IllegalArgumentException.class, () -> RedisEntryParserFactory.getRedisEntryParser(redisSinkConfig, statsDReporter, schema)); + assertEquals("Empty config SINK_REDIS_KEY_VALUE_DATA_FIELD_NAME found", illegalArgumentException.getMessage()); + } + + @Test + public void shouldThrowExceptionForEmptyListDataFieldName() { + when(redisSinkConfig.getSinkRedisDataType()).thenReturn(RedisSinkDataType.LIST); + when(redisSinkConfig.getSinkRedisListDataFieldName()).thenReturn(""); + IllegalArgumentException illegalArgumentException = + assertThrows(IllegalArgumentException.class, () -> RedisEntryParserFactory.getRedisEntryParser(redisSinkConfig, statsDReporter, schema)); + assertEquals("Empty config SINK_REDIS_LIST_DATA_FIELD_NAME found", illegalArgumentException.getMessage()); + } + + @Test + public void shouldThrowExceptionForEmptyRedisTemplate() { + when(redisSinkConfig.getSinkRedisKeyTemplate()).thenReturn(""); + IllegalArgumentException illegalArgumentException = + assertThrows(IllegalArgumentException.class, () -> RedisEntryParserFactory.getRedisEntryParser(redisSinkConfig, statsDReporter, schema)); + assertEquals("Template '' is invalid", illegalArgumentException.getMessage()); + } +} diff --git a/src/test/java/io/odpf/depot/redis/parsers/RedisHashSetEntryParserTest.java b/src/test/java/io/odpf/depot/redis/parsers/RedisHashSetEntryParserTest.java new file mode 100644 index 00000000..3c76a5fe --- /dev/null +++ b/src/test/java/io/odpf/depot/redis/parsers/RedisHashSetEntryParserTest.java @@ -0,0 +1,122 @@ +package io.odpf.depot.redis.parsers; + +import com.google.protobuf.Descriptors; +import io.odpf.depot.TestBookingLogMessage; +import io.odpf.depot.TestKey; +import io.odpf.depot.TestLocation; +import io.odpf.depot.config.RedisSinkConfig; +import io.odpf.depot.config.converter.JsonToPropertiesConverter; +import io.odpf.depot.message.OdpfMessage; +import io.odpf.depot.message.OdpfMessageSchema; +import io.odpf.depot.message.ParsedOdpfMessage; +import io.odpf.depot.message.SinkConnectorSchemaMessageMode; +import io.odpf.depot.message.proto.ProtoOdpfMessageParser; +import io.odpf.depot.metrics.StatsDReporter; +import io.odpf.depot.redis.client.entry.RedisEntry; +import io.odpf.depot.redis.client.entry.RedisHashSetFieldEntry; +import io.odpf.depot.redis.enums.RedisSinkDataType; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import java.io.IOException; +import java.util.*; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class RedisHashSetEntryParserTest { + private final Map descriptorsMap = new HashMap() {{ + put(String.format("%s", TestKey.class.getName()), TestKey.getDescriptor()); + put(String.format("%s", TestBookingLogMessage.class.getName()), TestBookingLogMessage.getDescriptor()); + put(String.format("%s", TestLocation.class.getName()), TestLocation.getDescriptor()); + }}; + @Mock + private RedisSinkConfig redisSinkConfig; + @Mock + private StatsDReporter statsDReporter; + private ParsedOdpfMessage parsedBookingMessage; + private ParsedOdpfMessage parsedOdpfKey; + private OdpfMessageSchema schemaBooking; + private OdpfMessageSchema schemaKey; + + private void redisSinkSetup(String field) throws IOException { + when(redisSinkConfig.getSinkRedisDataType()).thenReturn(RedisSinkDataType.HASHSET); + when(redisSinkConfig.getSinkRedisHashsetFieldToColumnMapping()).thenReturn(new JsonToPropertiesConverter().convert(null, field)); + when(redisSinkConfig.getSinkRedisKeyTemplate()).thenReturn("test-key"); + String schemaBookingClass = "io.odpf.depot.TestBookingLogMessage"; + String schemaKeyClass = "io.odpf.depot.TestKey"; + TestKey testKey = TestKey.newBuilder().setOrderNumber("ORDER-1-FROM-KEY").build(); + TestBookingLogMessage testBookingLogMessage = TestBookingLogMessage.newBuilder().setOrderNumber("booking-order-1").setCustomerTotalFareWithoutSurge(2000L).setAmountPaidByCash(12.3F).build(); + OdpfMessage bookingMessage = new OdpfMessage(testKey.toByteArray(), testBookingLogMessage.toByteArray()); + ProtoOdpfMessageParser odpfMessageParser = new ProtoOdpfMessageParser(redisSinkConfig, statsDReporter, null); + parsedBookingMessage = odpfMessageParser.parse(bookingMessage, SinkConnectorSchemaMessageMode.LOG_MESSAGE, schemaBookingClass); + parsedOdpfKey = odpfMessageParser.parse(bookingMessage, SinkConnectorSchemaMessageMode.LOG_KEY, schemaKeyClass); + schemaBooking = odpfMessageParser.getSchema(schemaBookingClass, descriptorsMap); + schemaKey = odpfMessageParser.getSchema(schemaKeyClass, descriptorsMap); + } + + @Test + public void shouldParseLongMessageForKey() throws IOException { + redisSinkSetup("{\"order_number\":\"ORDER_NUMBER_%d,customer_total_fare_without_surge\"}"); + RedisEntryParser redisHashSetEntryParser = RedisEntryParserFactory.getRedisEntryParser(redisSinkConfig, statsDReporter, schemaBooking); + List redisEntries = redisHashSetEntryParser.getRedisEntry(parsedBookingMessage); + RedisHashSetFieldEntry expectedEntry = new RedisHashSetFieldEntry("test-key", "ORDER_NUMBER_2000", "booking-order-1", null); + assertEquals(Collections.singletonList(expectedEntry), redisEntries); + } + + @Test + public void shouldParseLongMessageWithSpaceForKey() throws IOException { + redisSinkSetup("{\"order_number\":\"ORDER_NUMBER_%d, customer_total_fare_without_surge\"}"); + RedisEntryParser redisHashSetEntryParser = RedisEntryParserFactory.getRedisEntryParser(redisSinkConfig, statsDReporter, schemaBooking); + List redisEntries = redisHashSetEntryParser.getRedisEntry(parsedBookingMessage); + RedisHashSetFieldEntry expectedEntry = new RedisHashSetFieldEntry("test-key", "ORDER_NUMBER_2000", "booking-order-1", null); + assertEquals(Collections.singletonList(expectedEntry), redisEntries); + } + + @Test + public void shouldParseStringMessageForKey() throws IOException { + redisSinkSetup("{\"order_number\":\"ORDER_NUMBER_%s,order_number\"}"); + RedisEntryParser redisHashSetEntryParser = RedisEntryParserFactory.getRedisEntryParser(redisSinkConfig, statsDReporter, schemaBooking); + List redisEntries = redisHashSetEntryParser.getRedisEntry(parsedBookingMessage); + RedisHashSetFieldEntry expectedEntry = new RedisHashSetFieldEntry("test-key", "ORDER_NUMBER_booking-order-1", "booking-order-1", null); + assertEquals(Collections.singletonList(expectedEntry), redisEntries); + } + + @Test + public void shouldHandleStaticStringForKey() throws IOException { + redisSinkSetup("{\"order_number\":\"ORDER_NUMBER\"}"); + RedisEntryParser redisHashSetEntryParser = RedisEntryParserFactory.getRedisEntryParser(redisSinkConfig, statsDReporter, schemaBooking); + List redisEntries = redisHashSetEntryParser.getRedisEntry(parsedBookingMessage); + RedisHashSetFieldEntry expectedEntry = new RedisHashSetFieldEntry("test-key", "ORDER_NUMBER", "booking-order-1", null); + assertEquals(Collections.singletonList(expectedEntry), redisEntries); + } + + @Test + public void shouldThrowErrorForInvalidFormatForKey() throws IOException { + redisSinkSetup("{\"order_details\":\"ORDER_NUMBER%, order_number\"}"); + IllegalArgumentException e = Assert.assertThrows(IllegalArgumentException.class, () -> RedisEntryParserFactory.getRedisEntryParser(redisSinkConfig, statsDReporter, schemaBooking)); + assertEquals("Template is not valid, variables=1, validArgs=0, values=1", e.getMessage()); + } + + @Test + public void shouldThrowErrorForIncompatibleFormatForKey() throws IOException { + redisSinkSetup("{\"order_details\":\"order_number-%d, order_number\"}"); + RedisEntryParser redisHashSetEntryParser = RedisEntryParserFactory.getRedisEntryParser(redisSinkConfig, statsDReporter, schemaBooking); + IllegalFormatConversionException e = Assert.assertThrows(IllegalFormatConversionException.class, + () -> redisHashSetEntryParser.getRedisEntry(parsedBookingMessage)); + assertEquals("d != java.lang.String", e.getMessage()); + } + + @Test + public void shouldParseKeyWhenKafkaMessageParseModeSetToKey() throws IOException { + redisSinkSetup("{\"order_number\":\"ORDER_NUMBER\"}"); + RedisEntryParser redisHashSetEntryParser = RedisEntryParserFactory.getRedisEntryParser(redisSinkConfig, statsDReporter, schemaKey); + List redisEntries = redisHashSetEntryParser.getRedisEntry(parsedOdpfKey); + RedisHashSetFieldEntry expectedEntry = new RedisHashSetFieldEntry("test-key", "ORDER_NUMBER", "ORDER-1-FROM-KEY", null); + assertEquals(Collections.singletonList(expectedEntry), redisEntries); + } +} diff --git a/src/test/java/io/odpf/depot/redis/parsers/RedisKeyValueEntryParserTest.java b/src/test/java/io/odpf/depot/redis/parsers/RedisKeyValueEntryParserTest.java new file mode 100644 index 00000000..1c395949 --- /dev/null +++ b/src/test/java/io/odpf/depot/redis/parsers/RedisKeyValueEntryParserTest.java @@ -0,0 +1,77 @@ +package io.odpf.depot.redis.parsers; + +import com.google.protobuf.Descriptors; +import io.odpf.depot.*; +import io.odpf.depot.config.RedisSinkConfig; +import io.odpf.depot.exception.ConfigurationException; +import io.odpf.depot.message.*; +import io.odpf.depot.message.proto.ProtoOdpfMessageParser; +import io.odpf.depot.metrics.StatsDReporter; +import io.odpf.depot.redis.client.entry.RedisEntry; +import io.odpf.depot.redis.client.entry.RedisKeyValueEntry; +import io.odpf.depot.redis.enums.RedisSinkDataType; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class RedisKeyValueEntryParserTest { + @Mock + private RedisSinkConfig redisSinkConfig; + @Mock + private StatsDReporter statsDReporter; + private RedisEntryParser redisKeyValueEntryParser; + + private OdpfMessageSchema schema; + private ParsedOdpfMessage parsedOdpfMessage; + private final Map descriptorsMap = new HashMap() {{ + put(String.format("%s", TestKey.class.getName()), TestKey.getDescriptor()); + put(String.format("%s", TestMessage.class.getName()), TestMessage.getDescriptor()); + put(String.format("%s", TestNestedMessage.class.getName()), TestNestedMessage.getDescriptor()); + put(String.format("%s", TestNestedRepeatedMessage.class.getName()), TestNestedRepeatedMessage.getDescriptor()); + }}; + + private void redisSinkSetup(String template, String field) throws IOException { + when(redisSinkConfig.getSinkRedisDataType()).thenReturn(RedisSinkDataType.KEYVALUE); + when(redisSinkConfig.getSinkRedisKeyValueDataFieldName()).thenReturn(field); + when(redisSinkConfig.getSinkRedisKeyTemplate()).thenReturn(template); + ProtoOdpfMessageParser odpfMessageParser = new ProtoOdpfMessageParser(redisSinkConfig, statsDReporter, null); + String schemaClass = "io.odpf.depot.TestMessage"; + schema = odpfMessageParser.getSchema(schemaClass, descriptorsMap); + byte[] logMessage = TestMessage.newBuilder() + .setOrderNumber("xyz-order") + .setOrderDetails("new-eureka-order") + .build() + .toByteArray(); + OdpfMessage message = new OdpfMessage(null, logMessage); + parsedOdpfMessage = odpfMessageParser.parse(message, SinkConnectorSchemaMessageMode.LOG_MESSAGE, schemaClass); + redisKeyValueEntryParser = RedisEntryParserFactory.getRedisEntryParser(redisSinkConfig, statsDReporter, schema); + } + + @Test + public void shouldConvertParsedOdpfMessageToRedisKeyValueEntry() throws IOException { + redisSinkSetup("test-key", "order_details"); + List redisDataEntries = redisKeyValueEntryParser.getRedisEntry(parsedOdpfMessage); + RedisKeyValueEntry expectedEntry = new RedisKeyValueEntry("test-key", "new-eureka-order", null); + assertEquals(Collections.singletonList(expectedEntry), redisDataEntries); + } + + @Test + public void shouldThrowExceptionForInvalidKeyValueDataFieldName() throws IOException { + redisSinkSetup("test-key", "random-field"); + ConfigurationException configurationException = + assertThrows(ConfigurationException.class, () -> redisKeyValueEntryParser.getRedisEntry(parsedOdpfMessage)); + assertEquals("Invalid field config : random-field", configurationException.getMessage()); + } +} diff --git a/src/test/java/io/odpf/depot/redis/parsers/RedisListEntryParserTest.java b/src/test/java/io/odpf/depot/redis/parsers/RedisListEntryParserTest.java new file mode 100644 index 00000000..895abc09 --- /dev/null +++ b/src/test/java/io/odpf/depot/redis/parsers/RedisListEntryParserTest.java @@ -0,0 +1,82 @@ +package io.odpf.depot.redis.parsers; + +import com.google.protobuf.Descriptors; +import io.odpf.depot.TestKey; +import io.odpf.depot.TestMessage; +import io.odpf.depot.TestNestedMessage; +import io.odpf.depot.TestNestedRepeatedMessage; +import io.odpf.depot.config.RedisSinkConfig; +import io.odpf.depot.exception.ConfigurationException; +import io.odpf.depot.message.OdpfMessage; +import io.odpf.depot.message.OdpfMessageSchema; +import io.odpf.depot.message.ParsedOdpfMessage; +import io.odpf.depot.message.SinkConnectorSchemaMessageMode; +import io.odpf.depot.message.proto.ProtoOdpfMessageParser; +import io.odpf.depot.metrics.StatsDReporter; +import io.odpf.depot.redis.client.entry.RedisEntry; +import io.odpf.depot.redis.client.entry.RedisListEntry; +import io.odpf.depot.redis.enums.RedisSinkDataType; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class RedisListEntryParserTest { + private final Map descriptorsMap = new HashMap() {{ + put(String.format("%s", TestKey.class.getName()), TestKey.getDescriptor()); + put(String.format("%s", TestMessage.class.getName()), TestMessage.getDescriptor()); + put(String.format("%s", TestNestedMessage.class.getName()), TestNestedMessage.getDescriptor()); + put(String.format("%s", TestNestedRepeatedMessage.class.getName()), TestNestedRepeatedMessage.getDescriptor()); + }}; + @Mock + private RedisSinkConfig redisSinkConfig; + @Mock + private StatsDReporter statsDReporter; + private RedisEntryParser redisListEntryParser; + private OdpfMessageSchema schema; + private ParsedOdpfMessage parsedOdpfMessage; + + private void redisSinkSetup(String template, String field) throws IOException { + when(redisSinkConfig.getSinkRedisDataType()).thenReturn(RedisSinkDataType.LIST); + when(redisSinkConfig.getSinkRedisListDataFieldName()).thenReturn(field); + when(redisSinkConfig.getSinkRedisKeyTemplate()).thenReturn(template); + ProtoOdpfMessageParser odpfMessageParser = new ProtoOdpfMessageParser(redisSinkConfig, statsDReporter, null); + String schemaClass = "io.odpf.depot.TestMessage"; + schema = odpfMessageParser.getSchema(schemaClass, descriptorsMap); + byte[] logMessage = TestMessage.newBuilder() + .setOrderNumber("xyz-order") + .setOrderDetails("new-eureka-order") + .build() + .toByteArray(); + OdpfMessage message = new OdpfMessage(null, logMessage); + parsedOdpfMessage = odpfMessageParser.parse(message, SinkConnectorSchemaMessageMode.LOG_MESSAGE, schemaClass); + redisListEntryParser = RedisEntryParserFactory.getRedisEntryParser(redisSinkConfig, statsDReporter, schema); + } + + @Test + public void shouldConvertParsedOdpfMessageToRedisListEntry() throws IOException { + redisSinkSetup("test-key", "order_details"); + List redisDataEntries = redisListEntryParser.getRedisEntry(parsedOdpfMessage); + RedisListEntry expectedEntry = new RedisListEntry("test-key", "new-eureka-order", null); + assertEquals(Collections.singletonList(expectedEntry), redisDataEntries); + } + + @Test + public void shouldThrowExceptionForInvalidKeyValueDataFieldName() throws IOException { + redisSinkSetup("test-key", "random-field"); + ConfigurationException configurationException = + assertThrows(ConfigurationException.class, () -> redisListEntryParser.getRedisEntry(parsedOdpfMessage)); + assertEquals("Invalid field config : random-field", configurationException.getMessage()); + } +} diff --git a/src/test/java/io/odpf/depot/redis/parsers/RedisParserTest.java b/src/test/java/io/odpf/depot/redis/parsers/RedisParserTest.java new file mode 100644 index 00000000..84c38c72 --- /dev/null +++ b/src/test/java/io/odpf/depot/redis/parsers/RedisParserTest.java @@ -0,0 +1,126 @@ +package io.odpf.depot.redis.parsers; + +import com.google.protobuf.Descriptors; +import io.odpf.depot.TestKey; +import io.odpf.depot.TestMessage; +import io.odpf.depot.TestNestedMessage; +import io.odpf.depot.TestNestedRepeatedMessage; +import io.odpf.depot.common.Tuple; +import io.odpf.depot.config.RedisSinkConfig; +import io.odpf.depot.config.enums.SinkConnectorSchemaDataType; +import io.odpf.depot.error.ErrorType; +import io.odpf.depot.exception.ConfigurationException; +import io.odpf.depot.message.*; +import io.odpf.depot.message.proto.ProtoOdpfMessageParser; +import io.odpf.depot.message.proto.ProtoOdpfParsedMessage; +import io.odpf.depot.metrics.StatsDReporter; +import io.odpf.depot.redis.client.entry.RedisKeyValueEntry; +import io.odpf.depot.redis.enums.RedisSinkDataType; +import io.odpf.depot.redis.record.RedisRecord; +import io.odpf.depot.utils.MessageConfigUtils; +import io.odpf.stencil.Parser; +import io.odpf.stencil.StencilClientFactory; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class RedisParserTest { + private final List messages = new ArrayList<>(); + private final String schemaClass = "io.odpf.depot.TestMessage"; + private final Map descriptorsMap = new HashMap() {{ + put(String.format("%s", TestKey.class.getName()), TestKey.getDescriptor()); + put(String.format("%s", TestMessage.class.getName()), TestMessage.getDescriptor()); + put(String.format("%s", TestNestedMessage.class.getName()), TestNestedMessage.getDescriptor()); + put(String.format("%s", TestNestedRepeatedMessage.class.getName()), TestNestedRepeatedMessage.getDescriptor()); + }}; + @Mock + private RedisSinkConfig redisSinkConfig; + @Mock + private ProtoOdpfMessageParser odpfMessageParser; + @Mock + private StatsDReporter statsDReporter; + private RedisParser redisParser; + + @Before + public void setup() throws IOException { + when(redisSinkConfig.getSinkRedisDataType()).thenReturn(RedisSinkDataType.KEYVALUE); + when(redisSinkConfig.getSinkRedisKeyTemplate()).thenReturn("test-key"); + when(redisSinkConfig.getSinkRedisKeyValueDataFieldName()).thenReturn("order_number"); + when(redisSinkConfig.getSinkConnectorSchemaMessageMode()).thenReturn(SinkConnectorSchemaMessageMode.LOG_MESSAGE); + when(redisSinkConfig.getSinkConnectorSchemaProtoMessageClass()).thenReturn(schemaClass); + when(redisSinkConfig.getSinkConnectorSchemaDataType()).thenReturn(SinkConnectorSchemaDataType.PROTOBUF); + TestMessage message1 = TestMessage.newBuilder().setOrderNumber("test-order-1").setOrderDetails("ORDER-DETAILS-1").build(); + TestMessage message2 = TestMessage.newBuilder().setOrderNumber("test-order-2").setOrderDetails("ORDER-DETAILS-2").build(); + TestMessage message3 = TestMessage.newBuilder().setOrderNumber("test-order-3").setOrderDetails("ORDER-DETAILS-3").build(); + TestMessage message4 = TestMessage.newBuilder().setOrderNumber("test-order-4").setOrderDetails("ORDER-DETAILS-4").build(); + TestMessage message5 = TestMessage.newBuilder().setOrderNumber("test-order-5").setOrderDetails("ORDER-DETAILS-5").build(); + messages.add(new OdpfMessage(null, message1.toByteArray())); + messages.add(new OdpfMessage(null, message2.toByteArray())); + messages.add(new OdpfMessage(null, message3.toByteArray())); + messages.add(new OdpfMessage(null, message4.toByteArray())); + messages.add(new OdpfMessage(null, message5.toByteArray())); + } + + public void setupParserResponse() throws IOException { + Parser protoParser = StencilClientFactory.getClient().getParser(TestMessage.class.getName()); + for (OdpfMessage message : messages) { + ParsedOdpfMessage parsedOdpfMessage = new ProtoOdpfParsedMessage(protoParser.parse((byte[]) message.getLogMessage())); + when(odpfMessageParser.parse(message, SinkConnectorSchemaMessageMode.LOG_MESSAGE, schemaClass)).thenReturn(parsedOdpfMessage); + } + ProtoOdpfMessageParser messageParser = (ProtoOdpfMessageParser) OdpfMessageParserFactory.getParser(redisSinkConfig, statsDReporter); + Tuple modeAndSchema = MessageConfigUtils.getModeAndSchema(redisSinkConfig); + OdpfMessageSchema schema = messageParser.getSchema(modeAndSchema.getSecond(), descriptorsMap); + RedisEntryParser redisEntryParser = RedisEntryParserFactory.getRedisEntryParser(redisSinkConfig, statsDReporter, schema); + redisParser = new RedisParser(odpfMessageParser, redisEntryParser, modeAndSchema); + } + + @Test + public void shouldConvertOdpfMessageToRedisRecords() throws IOException { + setupParserResponse(); + List parsedRecords = redisParser.convert(messages); + Map> splitterRecords = parsedRecords.stream().collect(Collectors.partitioningBy(RedisRecord::isValid)); + List invalidRecords = splitterRecords.get(Boolean.FALSE); + List validRecords = splitterRecords.get(Boolean.TRUE); + assertEquals(5, validRecords.size()); + assertTrue(invalidRecords.isEmpty()); + List expectedRecords = new ArrayList<>(); + expectedRecords.add(new RedisRecord(new RedisKeyValueEntry("test-key", "test-order-1", null), 0L, null, "{}", true)); + expectedRecords.add(new RedisRecord(new RedisKeyValueEntry("test-key", "test-order-2", null), 1L, null, "{}", true)); + expectedRecords.add(new RedisRecord(new RedisKeyValueEntry("test-key", "test-order-3", null), 2L, null, "{}", true)); + expectedRecords.add(new RedisRecord(new RedisKeyValueEntry("test-key", "test-order-4", null), 3L, null, "{}", true)); + expectedRecords.add(new RedisRecord(new RedisKeyValueEntry("test-key", "test-order-5", null), 4L, null, "{}", true)); + IntStream.range(0, expectedRecords.size()).forEach(index -> assertEquals(expectedRecords.get(index).toString(), parsedRecords.get(index).toString())); + } + + @Test + public void shouldReportValidAndInvalidRecords() throws IOException { + setupParserResponse(); + when(odpfMessageParser.parse(messages.get(2), SinkConnectorSchemaMessageMode.LOG_MESSAGE, schemaClass)).thenThrow(new IOException("Error while parsing protobuf")); + when(odpfMessageParser.parse(messages.get(3), SinkConnectorSchemaMessageMode.LOG_MESSAGE, schemaClass)).thenThrow(new ConfigurationException("Invalid field config : INVALID")); + when(odpfMessageParser.parse(messages.get(4), SinkConnectorSchemaMessageMode.LOG_MESSAGE, schemaClass)).thenThrow(new IllegalArgumentException("Config REDIS_CONFIG is empty")); + List parsedRecords = redisParser.convert(messages); + Map> splitterRecords = parsedRecords.stream().collect(Collectors.partitioningBy(RedisRecord::isValid)); + List invalidRecords = splitterRecords.get(Boolean.FALSE); + List validRecords = splitterRecords.get(Boolean.TRUE); + assertEquals(2, validRecords.size()); + assertEquals(3, invalidRecords.size()); + assertEquals(ErrorType.DESERIALIZATION_ERROR, parsedRecords.get(2).getErrorInfo().getErrorType()); + assertEquals(ErrorType.UNKNOWN_FIELDS_ERROR, parsedRecords.get(3).getErrorInfo().getErrorType()); + assertEquals(ErrorType.DEFAULT_ERROR, parsedRecords.get(4).getErrorInfo().getErrorType()); + } +} diff --git a/src/test/java/io/odpf/depot/redis/parsers/TemplateTest.java b/src/test/java/io/odpf/depot/redis/parsers/TemplateTest.java new file mode 100644 index 00000000..53590b16 --- /dev/null +++ b/src/test/java/io/odpf/depot/redis/parsers/TemplateTest.java @@ -0,0 +1,123 @@ +package io.odpf.depot.redis.parsers; + +import com.google.protobuf.Descriptors; +import io.odpf.depot.TestBookingLogMessage; +import io.odpf.depot.TestKey; +import io.odpf.depot.TestLocation; +import io.odpf.depot.TestMessage; +import io.odpf.depot.config.RedisSinkConfig; +import io.odpf.depot.config.enums.SinkConnectorSchemaDataType; +import io.odpf.depot.message.OdpfMessage; +import io.odpf.depot.message.OdpfMessageParserFactory; +import io.odpf.depot.message.OdpfMessageSchema; +import io.odpf.depot.message.ParsedOdpfMessage; +import io.odpf.depot.message.proto.ProtoOdpfMessageParser; +import io.odpf.depot.message.proto.ProtoOdpfParsedMessage; +import io.odpf.depot.metrics.StatsDReporter; +import io.odpf.stencil.Parser; +import io.odpf.stencil.StencilClientFactory; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class TemplateTest { + @Mock + private RedisSinkConfig redisSinkConfig; + @Mock + private StatsDReporter statsDReporter; + private ParsedOdpfMessage parsedTestMessage; + private ParsedOdpfMessage parsedBookingMessage; + private OdpfMessageSchema schemaTest; + private OdpfMessageSchema schemaBooking; + + @Before + public void setUp() throws Exception { + TestKey testKey = TestKey.newBuilder().setOrderNumber("ORDER-1-FROM-KEY").build(); + TestBookingLogMessage testBookingLogMessage = TestBookingLogMessage.newBuilder().setOrderNumber("booking-order-1").setCustomerTotalFareWithoutSurge(2000L).setAmountPaidByCash(12.3F).build(); + TestMessage testMessage = TestMessage.newBuilder().setOrderNumber("test-order").setOrderDetails("ORDER-DETAILS").build(); + OdpfMessage message = new OdpfMessage(testKey.toByteArray(), testMessage.toByteArray()); + OdpfMessage bookingMessage = new OdpfMessage(testKey.toByteArray(), testBookingLogMessage.toByteArray()); + Map descriptorsMap = new HashMap() {{ + put(String.format("%s", TestKey.class.getName()), TestKey.getDescriptor()); + put(String.format("%s", TestMessage.class.getName()), TestMessage.getDescriptor()); + put(String.format("%s", TestBookingLogMessage.class.getName()), TestBookingLogMessage.getDescriptor()); + put(String.format("%s", TestLocation.class.getName()), TestLocation.getDescriptor()); + }}; + Parser protoParserTest = StencilClientFactory.getClient().getParser(TestMessage.class.getName()); + parsedTestMessage = new ProtoOdpfParsedMessage(protoParserTest.parse((byte[]) message.getLogMessage())); + Parser protoParserBooking = StencilClientFactory.getClient().getParser(TestBookingLogMessage.class.getName()); + parsedBookingMessage = new ProtoOdpfParsedMessage(protoParserBooking.parse((byte[]) bookingMessage.getLogMessage())); + when(redisSinkConfig.getSinkConnectorSchemaDataType()).thenReturn(SinkConnectorSchemaDataType.PROTOBUF); + ProtoOdpfMessageParser messageParser = (ProtoOdpfMessageParser) OdpfMessageParserFactory.getParser(redisSinkConfig, statsDReporter); + schemaTest = messageParser.getSchema("io.odpf.depot.TestMessage", descriptorsMap); + schemaBooking = messageParser.getSchema("io.odpf.depot.TestBookingLogMessage", descriptorsMap); + } + + @Test + public void shouldParseStringMessageForCollectionKeyTemplate() { + Template template = new Template("Test-%s,order_number"); + assertEquals("Test-test-order", template.parse(parsedTestMessage, schemaTest)); + } + + @Test + public void shouldParseStringMessageWithSpacesForCollectionKeyTemplate() { + Template template = new Template("Test-%s, order_number"); + assertEquals("Test-test-order", template.parse(parsedTestMessage, schemaTest)); + } + + @Test + public void shouldParseFloatMessageForCollectionKeyTemplate() { + Template template = new Template("Test-%.2f,amount_paid_by_cash"); + assertEquals("Test-12.30", template.parse(parsedBookingMessage, schemaBooking)); + } + + @Test + public void shouldParseLongMessageForCollectionKeyTemplate() { + Template template = new Template("Test-%d,customer_total_fare_without_surge"); + assertEquals("Test-2000", template.parse(parsedBookingMessage, schemaBooking)); + } + + @Test + public void shouldThrowExceptionForNullCollectionKeyTemplate() { + IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> new Template(null)); + assertEquals("Template 'null' is invalid", e.getMessage()); + } + + @Test + public void shouldThrowExceptionForEmptyCollectionKeyTemplate() { + IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> new Template("")); + assertEquals("Template '' is invalid", e.getMessage()); + } + + @Test + public void shouldAcceptStringForCollectionKey() { + Template template = new Template("Test"); + assertEquals("Test", template.parse(parsedBookingMessage, schemaBooking)); + } + + @Test + public void shouldNotAcceptStringWithPatternForCollectionKeyWithEmptyVariables() { + IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> new Template("Test-%s%d%b,t1,t2")); + Assert.assertEquals("Template is not valid, variables=3, validArgs=3, values=2", e.getMessage()); + + e = assertThrows(IllegalArgumentException.class, () -> new Template("Test-%s%s%y,order_number,order_details")); + Assert.assertEquals("Template is not valid, variables=3, validArgs=2, values=2", e.getMessage()); + } + + @Test + public void shouldAcceptStringWithPatternForCollectionKeyWithMultipleVariables() { + Template template = new Template("Test-%s::%s, order_number, order_details"); + assertEquals("Test-test-order::ORDER-DETAILS", template.parse(parsedTestMessage, schemaTest)); + } +} diff --git a/src/test/java/io/odpf/depot/redis/record/RedisRecordTest.java b/src/test/java/io/odpf/depot/redis/record/RedisRecordTest.java new file mode 100644 index 00000000..f3be35ea --- /dev/null +++ b/src/test/java/io/odpf/depot/redis/record/RedisRecordTest.java @@ -0,0 +1,74 @@ +package io.odpf.depot.redis.record; + +import io.odpf.depot.error.ErrorInfo; +import io.odpf.depot.error.ErrorType; +import io.odpf.depot.redis.client.entry.RedisEntry; +import io.odpf.depot.redis.client.response.RedisClusterResponse; +import io.odpf.depot.redis.client.response.RedisStandaloneResponse; +import io.odpf.depot.redis.ttl.RedisTtl; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import redis.clients.jedis.JedisCluster; +import redis.clients.jedis.Pipeline; + +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class RedisRecordTest { + @Mock + private RedisEntry redisEntry; + @Mock + private JedisCluster jedisCluster; + @Mock + private Pipeline jedisPipeline; + @Mock + private RedisTtl redisTtl; + + @Test + public void shouldSendUsingCLusterClient() { + RedisClusterResponse response = Mockito.mock(RedisClusterResponse.class); + when(redisEntry.send(jedisCluster, redisTtl)).thenReturn(response); + RedisRecord redisRecord = new RedisRecord(redisEntry, 0L, null, "METADATA", true); + RedisClusterResponse redisClusterResponse = redisRecord.send(jedisCluster, redisTtl); + Assert.assertEquals(response, redisClusterResponse); + } + + @Test + public void shouldSendUsingStandaloneClient() { + RedisStandaloneResponse standaloneResponse = Mockito.mock(RedisStandaloneResponse.class); + when(redisEntry.send(jedisPipeline, redisTtl)).thenReturn(standaloneResponse); + RedisRecord redisRecord = new RedisRecord(redisEntry, 0L, null, "METADATA", true); + RedisStandaloneResponse redisResponse = redisRecord.send(jedisPipeline, redisTtl); + Assert.assertEquals(standaloneResponse, redisResponse); + } + + @Test + public void shouldGetToString() { + when(redisEntry.toString()).thenReturn("RedisEntry REDIS ENTRY TO STRING"); + RedisRecord redisRecord = new RedisRecord(redisEntry, 0L, null, "METADATA", true); + Assert.assertEquals("Metadata METADATA RedisEntry REDIS ENTRY TO STRING", redisRecord.toString()); + } + + @Test + public void shouldGetRecordIndex() { + RedisRecord redisRecord = new RedisRecord(redisEntry, 0L, null, "METADATA", true); + Assert.assertEquals(new Long(0), redisRecord.getIndex()); + } + + @Test + public void shouldGetRecordErrorInfo() { + ErrorInfo errorInfo = new ErrorInfo(new Exception(""), ErrorType.DEFAULT_ERROR); + RedisRecord redisRecord = new RedisRecord(redisEntry, 0L, errorInfo, "METADATA", true); + Assert.assertEquals(errorInfo, redisRecord.getErrorInfo()); + } + + @Test + public void shouldGetRecordValidBoolean() { + RedisRecord redisRecord = new RedisRecord(redisEntry, 0L, null, "METADATA", true); + Assert.assertTrue(redisRecord.isValid()); + } +} diff --git a/src/test/java/io/odpf/depot/redis/ttl/DurationTTLTest.java b/src/test/java/io/odpf/depot/redis/ttl/DurationTTLTest.java new file mode 100644 index 00000000..45f3c1fa --- /dev/null +++ b/src/test/java/io/odpf/depot/redis/ttl/DurationTTLTest.java @@ -0,0 +1,41 @@ +package io.odpf.depot.redis.ttl; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; +import redis.clients.jedis.JedisCluster; +import redis.clients.jedis.Pipeline; + +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +@RunWith(MockitoJUnitRunner.class) +public class DurationTTLTest { + + private DurationTtl durationTTL; + + @Mock + private Pipeline pipeline; + + @Mock + private JedisCluster jedisCluster; + + @Before + public void setup() { + durationTTL = new DurationTtl(10); + } + + @Test + public void shouldSetTTLInSecondsForPipeline() { + durationTTL.setTtl(pipeline, "test-key"); + verify(pipeline, times(1)).expire("test-key", 10); + } + + @Test + public void shouldSetTTLInSecondsForCluster() { + durationTTL.setTtl(jedisCluster, "test-key"); + verify(jedisCluster, times(1)).expire("test-key", 10); + } +} diff --git a/src/test/java/io/odpf/depot/redis/ttl/ExactTimeTTLTest.java b/src/test/java/io/odpf/depot/redis/ttl/ExactTimeTTLTest.java new file mode 100644 index 00000000..086abbf5 --- /dev/null +++ b/src/test/java/io/odpf/depot/redis/ttl/ExactTimeTTLTest.java @@ -0,0 +1,39 @@ +package io.odpf.depot.redis.ttl; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; +import redis.clients.jedis.JedisCluster; +import redis.clients.jedis.Pipeline; + +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +@RunWith(MockitoJUnitRunner.class) +public class ExactTimeTTLTest { + + private ExactTimeTtl exactTimeTTL; + @Mock + private Pipeline pipeline; + + @Mock + private JedisCluster jedisCluster; + + @Before + public void setup() { + exactTimeTTL = new ExactTimeTtl(10000000L); + } + + @Test + public void shouldSetUnixTimeStampAsTTLForPipeline() { + exactTimeTTL.setTtl(pipeline, "test-key"); + verify(pipeline, times(1)).expireAt("test-key", 10000000L); + } + + @Test + public void shouldSetUnixTimeStampAsTTLForCluster() { + exactTimeTTL.setTtl(jedisCluster, "test-key"); + verify(jedisCluster, times(1)).expireAt("test-key", 10000000L); + } +} diff --git a/src/test/java/io/odpf/depot/redis/ttl/RedisTtlFactoryTest.java b/src/test/java/io/odpf/depot/redis/ttl/RedisTtlFactoryTest.java new file mode 100644 index 00000000..dfb3c776 --- /dev/null +++ b/src/test/java/io/odpf/depot/redis/ttl/RedisTtlFactoryTest.java @@ -0,0 +1,60 @@ +package io.odpf.depot.redis.ttl; + +import io.odpf.depot.config.RedisSinkConfig; +import io.odpf.depot.exception.ConfigurationException; +import io.odpf.depot.redis.enums.RedisSinkTtlType; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import static org.mockito.Mockito.when; +@RunWith(MockitoJUnitRunner.class) +public class RedisTtlFactoryTest { + + @Mock + private RedisSinkConfig redisSinkConfig; + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + @Before + public void setup() { + when(redisSinkConfig.getSinkRedisTtlType()).thenReturn(RedisSinkTtlType.DISABLE); + } + + @Test + public void shouldReturnNoTTLIfNothingGiven() { + RedisTtl redisTTL = RedisTTLFactory.getTTl(redisSinkConfig); + Assert.assertEquals(redisTTL.getClass(), NoRedisTtl.class); + } + + @Test + public void shouldReturnExactTimeTTL() { + when(redisSinkConfig.getSinkRedisTtlType()).thenReturn(RedisSinkTtlType.EXACT_TIME); + when(redisSinkConfig.getSinkRedisTtlValue()).thenReturn(100L); + RedisTtl redisTTL = RedisTTLFactory.getTTl(redisSinkConfig); + Assert.assertEquals(redisTTL.getClass(), ExactTimeTtl.class); + } + + @Test + public void shouldReturnDurationTTL() { + when(redisSinkConfig.getSinkRedisTtlType()).thenReturn(RedisSinkTtlType.DURATION); + when(redisSinkConfig.getSinkRedisTtlValue()).thenReturn(100L); + RedisTtl redisTTL = RedisTTLFactory.getTTl(redisSinkConfig); + Assert.assertEquals(redisTTL.getClass(), DurationTtl.class); + } + + @Test + public void shouldThrowExceptionInCaseOfInvalidConfiguration() { + expectedException.expect(ConfigurationException.class); + expectedException.expectMessage("Provide a positive TTL value"); + when(redisSinkConfig.getSinkRedisTtlValue()).thenReturn(-1L); + when(redisSinkConfig.getSinkRedisTtlType()).thenReturn(RedisSinkTtlType.DURATION); + RedisTTLFactory.getTTl(redisSinkConfig); + } +} diff --git a/src/test/java/io/odpf/depot/redis/util/RedisSinkUtilsTest.java b/src/test/java/io/odpf/depot/redis/util/RedisSinkUtilsTest.java new file mode 100644 index 00000000..79f94ed9 --- /dev/null +++ b/src/test/java/io/odpf/depot/redis/util/RedisSinkUtilsTest.java @@ -0,0 +1,71 @@ +package io.odpf.depot.redis.util; + +import io.odpf.depot.error.ErrorInfo; +import io.odpf.depot.error.ErrorType; +import io.odpf.depot.metrics.Instrumentation; +import io.odpf.depot.metrics.StatsDReporter; +import io.odpf.depot.redis.client.entry.RedisListEntry; +import io.odpf.depot.redis.client.response.RedisClusterResponse; +import io.odpf.depot.redis.client.response.RedisResponse; +import io.odpf.depot.redis.record.RedisRecord; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +@RunWith(MockitoJUnitRunner.class) +public class RedisSinkUtilsTest { + @Mock + private StatsDReporter statsDReporter; + + @Test + public void shouldGetErrorsFromResponse() { + List records = new ArrayList<>(); + records.add(new RedisRecord(new RedisListEntry("key1", "val1", null), 1L, null, null, true)); + records.add(new RedisRecord(new RedisListEntry("key1", "val1", null), 4L, null, null, true)); + records.add(new RedisRecord(new RedisListEntry("key1", "val1", null), 7L, null, null, true)); + records.add(new RedisRecord(new RedisListEntry("key1", "val1", null), 10L, null, null, true)); + records.add(new RedisRecord(new RedisListEntry("key1", "val1", null), 15L, null, null, true)); + List responses = new ArrayList<>(); + responses.add(new RedisClusterResponse("LPUSH", "OK", null)); + responses.add(new RedisClusterResponse("FAILED AT 4")); + responses.add(new RedisClusterResponse("FAILED AT 7")); + responses.add(new RedisClusterResponse("FAILED AT 10")); + responses.add(new RedisClusterResponse("LPUSH", "OK", null)); + Map errors = RedisSinkUtils.getErrorsFromResponse(records, responses, new Instrumentation(statsDReporter, RedisSinkUtils.class)); + Assert.assertEquals(3, errors.size()); + Assert.assertEquals("FAILED AT 4", errors.get(4L).getException().getMessage()); + Assert.assertEquals("FAILED AT 7", errors.get(7L).getException().getMessage()); + Assert.assertEquals("FAILED AT 10", errors.get(10L).getException().getMessage()); + Assert.assertEquals(ErrorType.DEFAULT_ERROR, errors.get(4L).getErrorType()); + Assert.assertEquals(ErrorType.DEFAULT_ERROR, errors.get(7L).getErrorType()); + Assert.assertEquals(ErrorType.DEFAULT_ERROR, errors.get(10L).getErrorType()); + } + + @Test + public void shouldGetEmptyMapWhenNoErrors() { + List records = new ArrayList<>(); + records.add(new RedisRecord(new RedisListEntry("key1", "val1", null), 1L, null, null, true)); + records.add(new RedisRecord(new RedisListEntry("key1", "val1", null), 4L, null, null, true)); + records.add(new RedisRecord(new RedisListEntry("key1", "val1", null), 7L, null, null, true)); + records.add(new RedisRecord(new RedisListEntry("key1", "val1", null), 10L, null, null, true)); + records.add(new RedisRecord(new RedisListEntry("key1", "val1", null), 15L, null, null, true)); + List responses = new ArrayList<>(); + responses.add(Mockito.mock(RedisResponse.class)); + responses.add(Mockito.mock(RedisResponse.class)); + responses.add(Mockito.mock(RedisResponse.class)); + responses.add(Mockito.mock(RedisResponse.class)); + responses.add(Mockito.mock(RedisResponse.class)); + responses.forEach(response -> { + Mockito.when(response.isFailed()).thenReturn(false); + }); + Map errors = RedisSinkUtils.getErrorsFromResponse(records, responses, new Instrumentation(statsDReporter, RedisSinkUtils.class)); + Assert.assertTrue(errors.isEmpty()); + } +} diff --git a/src/test/java/io/odpf/depot/utils/JsonUtilsTest.java b/src/test/java/io/odpf/depot/utils/JsonUtilsTest.java new file mode 100644 index 00000000..37aea662 --- /dev/null +++ b/src/test/java/io/odpf/depot/utils/JsonUtilsTest.java @@ -0,0 +1,99 @@ +package io.odpf.depot.utils; + +import io.odpf.depot.config.OdpfSinkConfig; +import org.junit.Assert; +import org.junit.Test; +import org.json.JSONObject; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class JsonUtilsTest { + @Mock + private OdpfSinkConfig odpfSinkConfig; + + void setSinkConfigs(boolean stringModeEnabled) { + when(odpfSinkConfig.getSinkConnectorSchemaJsonParserStringModeEnabled()).thenReturn(stringModeEnabled); + } + + @Test + public void shouldParseSimpleJsonWhenStringModeEnabled() { + setSinkConfigs(true); + JSONObject expectedJson = new JSONObject(); + expectedJson.put("name", "foo"); + expectedJson.put("num", "0371480"); + expectedJson.put("balance", "100"); + expectedJson.put("is_vip", "YES"); + byte[] payload = expectedJson.toString().getBytes(); + JSONObject parsedJson = JsonUtils.getJsonObject(odpfSinkConfig, payload); + Assert.assertTrue(parsedJson.similar(expectedJson)); + } + + @Test + public void shouldCastAllTypeToStringWhenStringModeEnabled() { + setSinkConfigs(true); + JSONObject originalJson = new JSONObject(); + originalJson.put("name", "foo"); + originalJson.put("num", new Integer(100)); + originalJson.put("balance", new Double(1000.21)); + originalJson.put("is_vip", Boolean.TRUE); + byte[] payload = originalJson.toString().getBytes(); + JSONObject parsedJson = JsonUtils.getJsonObject(odpfSinkConfig, payload); + JSONObject stringJson = new JSONObject(); + stringJson.put("name", "foo"); + stringJson.put("num", "100"); + stringJson.put("balance", "1000.21"); + stringJson.put("is_vip", "true"); + Assert.assertTrue(parsedJson.similar(stringJson)); + } + + @Test + public void shouldThrowExceptionForNestedJsonWhenStringModeEnabled() { + setSinkConfigs(true); + JSONObject nestedJsonField = new JSONObject(); + nestedJsonField.put("name", "foo"); + nestedJsonField.put("num", "0371480"); + nestedJsonField.put("balance", "100"); + nestedJsonField.put("is_vip", "YES"); + JSONObject nestedJson = new JSONObject(); + nestedJson.put("ID", 1); + nestedJson.put("nestedField", nestedJsonField); + byte[] payload = nestedJson.toString().getBytes(); + UnsupportedOperationException exception = assertThrows(UnsupportedOperationException.class, + () -> JsonUtils.getJsonObject(odpfSinkConfig, payload)); + assertEquals("nested json structure not supported yet", exception.getMessage()); + } + + @Test + public void shouldParseSimpleJsonWhenStringModeDisabled() { + setSinkConfigs(false); + JSONObject expectedJson = new JSONObject(); + expectedJson.put("name", "foo"); + expectedJson.put("num", new Integer(100)); + expectedJson.put("balance", new Double(1000.21)); + expectedJson.put("is_vip", Boolean.TRUE); + byte[] payload = expectedJson.toString().getBytes(); + JSONObject parsedJson = JsonUtils.getJsonObject(odpfSinkConfig, payload); + Assert.assertTrue(parsedJson.similar(expectedJson)); + } + + @Test + public void shouldParseNestedJsonWhenStringModeDisabled() { + setSinkConfigs(false); + JSONObject nestedJsonField = new JSONObject(); + nestedJsonField.put("name", "foo"); + nestedJsonField.put("num", new Integer(100)); + nestedJsonField.put("balance", new Double(1000.21)); + nestedJsonField.put("is_vip", Boolean.TRUE); + JSONObject nestedJson = new JSONObject(); + nestedJson.put("ID", 1); + nestedJson.put("nestedField", nestedJsonField); + byte[] payload = nestedJson.toString().getBytes(); + JSONObject parsedJson = JsonUtils.getJsonObject(odpfSinkConfig, payload); + Assert.assertTrue(parsedJson.similar(nestedJson)); + } +} diff --git a/src/test/java/io/odpf/depot/utils/StringUtilsTest.java b/src/test/java/io/odpf/depot/utils/StringUtilsTest.java new file mode 100644 index 00000000..b8c94a5b --- /dev/null +++ b/src/test/java/io/odpf/depot/utils/StringUtilsTest.java @@ -0,0 +1,33 @@ +package io.odpf.depot.utils; + +import org.junit.Assert; +import org.junit.Test; + +public class StringUtilsTest { + + @Test + public void shouldReturnValidArgumentsForStringFormat() { + Assert.assertEquals(0, StringUtils.countVariables("test")); + Assert.assertEquals(0, StringUtils.countVariables("")); + Assert.assertEquals(1, StringUtils.countVariables("test%dtest")); + Assert.assertEquals(2, StringUtils.countVariables("test%dtest%ttest")); + Assert.assertEquals(5, StringUtils.countVariables("test%dtest%ttest dskladja %s ds %d sdajk %b")); + } + + @Test + public void shouldReturnCharacterCount() { + Assert.assertEquals(0, StringUtils.count("test", 'i')); + Assert.assertEquals(0, StringUtils.count("", '5')); + Assert.assertEquals(2, StringUtils.count("test", 't')); + Assert.assertEquals(1, StringUtils.count("test", 'e')); + Assert.assertEquals(1, StringUtils.count("test", 's')); + Assert.assertEquals(0, StringUtils.count("test", '%')); + } + + @Test + public void shouldReturnValidArgsAndCharacters() { + String testString = "test%s%ddjaklsjd%%%%s%y%d"; + Assert.assertEquals(8, StringUtils.count(testString, '%')); + Assert.assertEquals(4, StringUtils.countVariables(testString)); + } +}