Skip to content

Commit 269dd64

Browse files
DATAREDIS-803 - Work around Redis parameter limitation
Redis has a [limitation of 1024*1024 parameters](https://github.com/antirez/redis/blob/4.0.9/src/networking.c#L1200) for bulk operations. To insert more than 1024*1024/2-1 entries with putAll(), they need to be split up in multiple HMSET commands. To reveive more than 1024*1024-1 entries with entrySet(), we can directly use the HGETALL command instead of first fetching the keys with HKEYS and then fetching the values with HMGET.
1 parent e0b73c7 commit 269dd64

File tree

4 files changed

+34
-34
lines changed

4 files changed

+34
-34
lines changed

src/main/java/org/springframework/data/redis/core/DefaultHashOperations.java

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -15,13 +15,8 @@
1515
*/
1616
package org.springframework.data.redis.core;
1717

18-
import java.util.Collection;
19-
import java.util.Collections;
20-
import java.util.LinkedHashMap;
21-
import java.util.List;
22-
import java.util.Map;
18+
import java.util.*;
2319
import java.util.Map.Entry;
24-
import java.util.Set;
2520

2621
import org.springframework.core.convert.converter.Converter;
2722
import org.springframework.lang.Nullable;
@@ -141,16 +136,22 @@ public void putAll(K key, Map<? extends HK, ? extends HV> m) {
141136

142137
byte[] rawKey = rawKey(key);
143138

144-
Map<byte[], byte[]> hashes = new LinkedHashMap<>(m.size());
139+
int size = Math.min(RedisTemplate.REDIS_MAX_ARGS / 2 - 1, m.size());
140+
Map<byte[], byte[]> hashes = new LinkedHashMap<>(size);
145141

146-
for (Map.Entry<? extends HK, ? extends HV> entry : m.entrySet()) {
142+
Iterator<? extends Entry<? extends HK, ? extends HV>> entries = m.entrySet().iterator();
143+
while (entries.hasNext()) {
144+
Entry<? extends HK, ? extends HV> entry = entries.next();
147145
hashes.put(rawHashKey(entry.getKey()), rawHashValue(entry.getValue()));
148-
}
149146

150-
execute(connection -> {
151-
connection.hMSet(rawKey, hashes);
152-
return null;
153-
}, true);
147+
if (!entries.hasNext() || hashes.size() == size) {
148+
execute(connection -> {
149+
connection.hMSet(rawKey, hashes);
150+
return null;
151+
}, true);
152+
hashes.clear();
153+
}
154+
}
154155
}
155156

156157
/*

src/main/java/org/springframework/data/redis/core/RedisTemplate.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,8 @@
8585
*/
8686
public class RedisTemplate<K, V> extends RedisAccessor implements RedisOperations<K, V>, BeanClassLoaderAware {
8787

88+
public static final int REDIS_MAX_ARGS = 1024 * 1024;
89+
8890
private boolean enableTransactionSupport = false;
8991
private boolean exposeConnection = false;
9092
private boolean initialized = false;

src/main/java/org/springframework/data/redis/support/collections/DefaultRedisMap.java

Lines changed: 1 addition & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -155,20 +155,7 @@ public boolean containsValue(Object value) {
155155
*/
156156
@Override
157157
public Set<java.util.Map.Entry<K, V>> entrySet() {
158-
159-
Set<K> keySet = keySet();
160-
checkResult(keySet);
161-
Collection<V> multiGet = hashOps.multiGet(keySet);
162-
163-
Iterator<K> keys = keySet.iterator();
164-
Iterator<V> values = multiGet.iterator();
165-
166-
Set<Map.Entry<K, V>> entries = new LinkedHashSet<>();
167-
while (keys.hasNext()) {
168-
entries.add(new DefaultRedisMapEntry(keys.next(), values.next()));
169-
}
170-
171-
return entries;
158+
return hashOps.entries().entrySet();
172159
}
173160

174161
/*

src/test/java/org/springframework/data/redis/support/collections/AbstractRedisMapTests.java

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -22,14 +22,8 @@
2222

2323
import java.io.IOException;
2424
import java.text.DecimalFormat;
25-
import java.util.ArrayList;
26-
import java.util.Collection;
27-
import java.util.Collections;
28-
import java.util.LinkedHashMap;
29-
import java.util.LinkedHashSet;
30-
import java.util.Map;
25+
import java.util.*;
3126
import java.util.Map.Entry;
32-
import java.util.Set;
3327

3428
import org.junit.After;
3529
import org.junit.AfterClass;
@@ -396,6 +390,22 @@ public void testEntrySet() {
396390
assertThat(values, not(hasItem(v2)));
397391
}
398392

393+
@Test
394+
public void testBigEntrySet() {
395+
Set<Entry<K, V>> entries = map.entrySet();
396+
assertTrue(entries.isEmpty());
397+
398+
Map<K, V> m = new HashMap<>();
399+
for (int i = 0; i < RedisTemplate.REDIS_MAX_ARGS - 1; i++) {
400+
m.put(getKey(), getValue());
401+
}
402+
map.putAll(m);
403+
404+
entries = map.entrySet();
405+
406+
assertEquals(RedisTemplate.REDIS_MAX_ARGS - 1, entries.size());
407+
}
408+
399409
@Test
400410
public void testPutIfAbsent() {
401411

0 commit comments

Comments
 (0)