diff --git a/gradle.properties b/gradle.properties index 1068455f..8356a86c 100644 --- a/gradle.properties +++ b/gradle.properties @@ -11,7 +11,6 @@ jacocoVersion = 0.8.11 kordampBuildVersion = 3.4.0 kordampPluginVersion = 0.54.0 -globVersion = 0.9.0 lettucemodVersion = 3.7.3 testcontainersRedisVersion = 2.2.2 diff --git a/subprojects/spring-batch-redis-core/src/main/java/com/redis/spring/batch/reader/ScanSizeEstimator.java b/subprojects/spring-batch-redis-core/src/main/java/com/redis/spring/batch/reader/ScanSizeEstimator.java deleted file mode 100644 index 27d84db1..00000000 --- a/subprojects/spring-batch-redis-core/src/main/java/com/redis/spring/batch/reader/ScanSizeEstimator.java +++ /dev/null @@ -1,134 +0,0 @@ -package com.redis.spring.batch.reader; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.function.LongSupplier; -import java.util.function.Predicate; -import java.util.stream.Collectors; - -import org.springframework.util.StringUtils; - -import com.redis.lettucemod.api.StatefulRedisModulesConnection; -import com.redis.lettucemod.api.async.RedisModulesAsyncCommands; -import com.redis.lettucemod.util.RedisModulesUtils; -import com.redis.spring.batch.OperationExecutor; -import com.redis.spring.batch.util.BatchUtils; - -import io.lettuce.core.AbstractRedisClient; -import io.lettuce.core.RedisFuture; - -public class ScanSizeEstimator implements LongSupplier { - - public static final long UNKNOWN_SIZE = -1; - public static final int DEFAULT_SAMPLES = 100; - - private final AbstractRedisClient client; - - private int samples = DEFAULT_SAMPLES; - private String keyPattern; - private String keyType; - - public ScanSizeEstimator(AbstractRedisClient client) { - this.client = client; - } - - public String getKeyPattern() { - return keyPattern; - } - - public void setKeyPattern(String keyPattern) { - this.keyPattern = keyPattern; - } - - public String getKeyType() { - return keyType; - } - - public void setKeyType(String keyType) { - this.keyType = keyType; - } - - public int getSamples() { - return samples; - } - - public void setSamples(int samples) { - this.samples = samples; - } - - /** - * Estimates the number of keys that match the given pattern and type. - * - * @return Estimated number of keys matching the given pattern and type. Returns - * null if database is empty or any error occurs - * @throws IOException if script execution exception happens during estimation - */ - @Override - public long getAsLong() { - try (StatefulRedisModulesConnection connection = RedisModulesUtils.connection(client)) { - Long dbsize = connection.sync().dbsize(); - if (dbsize == null) { - return UNKNOWN_SIZE; - } - if (!StringUtils.hasLength(keyPattern) && !StringUtils.hasLength(keyType)) { - return dbsize; - } - RedisModulesAsyncCommands commands = connection.async(); - try { - connection.setAutoFlushCommands(false); - List> keyFutures = new ArrayList<>(); - for (int index = 0; index < samples; index++) { - keyFutures.add(commands.randomkey()); - } - connection.flushCommands(); - List keys = OperationExecutor.getAll(connection.getTimeout(), keyFutures); - List> typeFutures = keys.stream().map(commands::type).collect(Collectors.toList()); - connection.flushCommands(); - List types = OperationExecutor.getAll(connection.getTimeout(), typeFutures); - Predicate matchPredicate = matchPredicate(); - Predicate typePredicate = typePredicate(); - int total = 0; - int matchCount = 0; - Iterator keyIterator = keys.iterator(); - Iterator typeIterator = types.iterator(); - while (keyIterator.hasNext()) { - String key = keyIterator.next(); - if (!typeIterator.hasNext()) { - throw new IllegalStateException("Could not find type for key " + key); - } - String type = typeIterator.next(); - total++; - if (matchPredicate.test(key) && typePredicate.test(type)) { - matchCount++; - } - } - double matchRate = total == 0 ? 0 : (double) matchCount / total; - return Math.round(dbsize * matchRate); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } catch (Exception e) { - // Ignore and return unknown size - } finally { - connection.setAutoFlushCommands(true); - } - } - return UNKNOWN_SIZE; - } - - private Predicate matchPredicate() { - if (StringUtils.hasLength(keyPattern)) { - return BatchUtils.globPredicate(keyPattern); - } - return s -> true; - } - - private Predicate typePredicate() { - if (StringUtils.hasLength(keyType)) { - return keyType::equalsIgnoreCase; - } - return s -> true; - } - -} diff --git a/subprojects/spring-batch-redis-test/src/test/java/com/redis/spring/batch/test/BatchTests.java b/subprojects/spring-batch-redis-test/src/test/java/com/redis/spring/batch/test/BatchTests.java index 3a206d40..0513f0f7 100644 --- a/subprojects/spring-batch-redis-test/src/test/java/com/redis/spring/batch/test/BatchTests.java +++ b/subprojects/spring-batch-redis-test/src/test/java/com/redis/spring/batch/test/BatchTests.java @@ -68,7 +68,6 @@ import com.redis.spring.batch.reader.KeyNotificationItemReader; import com.redis.spring.batch.reader.MemKeyValue; import com.redis.spring.batch.reader.MemKeyValueRead.ValueType; -import com.redis.spring.batch.reader.ScanSizeEstimator; import com.redis.spring.batch.reader.StreamItemReader; import com.redis.spring.batch.reader.StreamItemReader.AckPolicy; import com.redis.spring.batch.util.BatchUtils; @@ -311,19 +310,6 @@ void compareStatus(TestInfo info) throws Exception { assertEquals(deleted, comparisons.stream().filter(c -> c.getStatus() == Status.MISSING).count()); } - @Test - void estimateScanSize(TestInfo info) throws Exception { - GeneratorItemReader gen = generator(3000, Item.Type.HASH, Item.Type.STRING); - generate(info, gen); - long expectedCount = redisCommands.dbsize(); - ScanSizeEstimator estimator = new ScanSizeEstimator(redisClient); - estimator.setKeyPattern(GeneratorItemReader.DEFAULT_KEYSPACE + ":*"); - estimator.setSamples(300); - assertEquals(expectedCount, estimator.getAsLong(), expectedCount / 10); - estimator.setKeyType(DataType.HASH.getString()); - assertEquals(expectedCount / 2, estimator.getAsLong(), expectedCount / 10); - } - @Test void readStruct(TestInfo info) throws Exception { generate(info, generator(73)); diff --git a/subprojects/spring-batch-redis-util/spring-batch-redis-util.gradle b/subprojects/spring-batch-redis-util/spring-batch-redis-util.gradle index 549496bd..cb98289d 100644 --- a/subprojects/spring-batch-redis-util/spring-batch-redis-util.gradle +++ b/subprojects/spring-batch-redis-util/spring-batch-redis-util.gradle @@ -1,5 +1,4 @@ dependencies { api 'org.springframework:spring-core' implementation group: 'com.redis', name: 'lettucemod', version: lettucemodVersion - implementation group: 'com.hrakaroo', name: 'glob', version: globVersion } \ No newline at end of file diff --git a/subprojects/spring-batch-redis-util/src/main/java/com/redis/spring/batch/util/BatchUtils.java b/subprojects/spring-batch-redis-util/src/main/java/com/redis/spring/batch/util/BatchUtils.java index 5a6196ac..c8183153 100644 --- a/subprojects/spring-batch-redis-util/src/main/java/com/redis/spring/batch/util/BatchUtils.java +++ b/subprojects/spring-batch-redis-util/src/main/java/com/redis/spring/batch/util/BatchUtils.java @@ -5,12 +5,10 @@ import java.io.InputStreamReader; import java.nio.ByteBuffer; import java.util.function.Function; -import java.util.function.Predicate; import java.util.function.Supplier; import org.springframework.util.FileCopyUtils; -import com.hrakaroo.glob.GlobPattern; import com.redis.lettucemod.RedisModulesClient; import com.redis.lettucemod.api.StatefulRedisModulesConnection; import com.redis.lettucemod.cluster.RedisModulesClusterClient; @@ -83,7 +81,4 @@ public static StatefulRedisModulesConnection connection(RedisModule return connection; } - public static Predicate globPredicate(String match) { - return GlobPattern.compile(match)::matches; - } } diff --git a/subprojects/spring-batch-redis-util/src/test/java/com/redis/spring/batch/core/PredicatesTests.java b/subprojects/spring-batch-redis-util/src/test/java/com/redis/spring/batch/core/PredicatesTests.java deleted file mode 100644 index b9d9cc08..00000000 --- a/subprojects/spring-batch-redis-util/src/test/java/com/redis/spring/batch/core/PredicatesTests.java +++ /dev/null @@ -1,45 +0,0 @@ -package com.redis.spring.batch.core; - -import static com.redis.spring.batch.util.BatchUtils.globPredicate; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.util.function.Predicate; - -import org.junit.jupiter.api.Test; - -class PredicatesTests { - - @Test - void include() { - Predicate foo = globPredicate("foo"); - assertTrue(foo.test("foo")); - assertFalse(foo.test("bar")); - Predicate fooStar = globPredicate("foo*"); - assertTrue(fooStar.test("foobar")); - assertFalse(fooStar.test("barfoo")); - } - - @Test - void exclude() { - Predicate foo = globPredicate("foo").negate(); - assertFalse(foo.test("foo")); - assertTrue(foo.test("foa")); - Predicate fooStar = globPredicate("foo*").negate(); - assertFalse(fooStar.test("foobar")); - assertTrue(fooStar.test("barfoo")); - } - - @Test - void includeAndExclude() { - Predicate foo1 = globPredicate("foo1").and(globPredicate("foo").negate()); - assertFalse(foo1.test("foo")); - assertFalse(foo1.test("bar")); - assertTrue(foo1.test("foo1")); - Predicate foo1Star = globPredicate("foo").and(globPredicate("foo1*").negate()); - assertTrue(foo1Star.test("foo")); - assertFalse(foo1Star.test("bar")); - assertFalse(foo1Star.test("foo1")); - } - -}