New Redis integration #3018 #3100

docs/configs/janusgraph-cfg.md

@@ -18,10 +18,17 @@ Configuration options that modify JanusGraph's caching behavior
 
 | Name | Description | Datatype | Default Value | Mutability |
 | ---- | ---- | ---- | ---- | ---- |
+| cache.cache-type | Enable or disable Redis cache (redis/inmemory) | String | redis | MASKABLE |
 | cache.db-cache | Whether to enable JanusGraph's database-level cache, which is shared across all transactions. Enabling this option speeds up traversals by holding hot graph elements in memory, but also increases the likelihood of reading stale data.  Disabling it forces each transaction to independently fetch graph elements from storage before reading/writing them. | Boolean | false | MASKABLE |
 | cache.db-cache-clean-wait | How long, in milliseconds, database-level cache will keep entries after flushing them.  This option is only useful on distributed storage backends that are capable of acknowledging writes without necessarily making them immediately visible. | Integer | 50 | MASKABLE |
 | cache.db-cache-size | Size of JanusGraph's database level cache.  Values between 0 and 1 are interpreted as a percentage of VM heap, while larger values are interpreted as an absolute size in bytes. | Double | 0.3 | MASKABLE |
 | cache.db-cache-time | Default expiration time, in milliseconds, for entries in the database-level cache. Entries are evicted when they reach this age even if the cache has room to spare. Set to 0 to disable expiration (cache entries live forever or until memory pressure triggers eviction when set to 0). | Long | 10000 | MASKABLE |
+| cache.redis-cache-connectTimeout | Timeout during connecting to any Redis server. | Integer | 1000 | MASKABLE |
+| cache.redis-cache-connectionMinimumIdleSize | Minimum idle Redis connection amount. | Integer | 24 | MASKABLE |
+| cache.redis-cache-connectionPoolSize | Redis connection maximum pool size. | Integer | 64 | MASKABLE |
+| cache.redis-cache-host | Redis host name | String | localhost | MASKABLE |
+| cache.redis-cache-keepAlive | Enables TCP keepAlive for connection. | Boolean | true | MASKABLE |
+| cache.redis-cache-port | Redis host port | Integer | 6379 | MASKABLE |
 | cache.tx-cache-size | Maximum size of the transaction-level cache of recently-used vertices. | Integer | 20000 | MASKABLE |
 | cache.tx-dirty-size | Initial size of the transaction-level cache of uncommitted dirty vertices. This is a performance hint for write-heavy, performance-sensitive transactional workloads. If set, it should roughly match the median vertices modified per transaction. | Integer | (no default value) | MASKABLE |
 

janusgraph-core/pom.xml

@@ -20,6 +20,54 @@
             <artifactId>janusgraph-driver</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>de.ruedigermoeller</groupId>
+            <artifactId>fst</artifactId>
+            <version>2.56</version>
+        </dependency>
+        <dependency>
+            <groupId>org.redisson</groupId>
+            <artifactId>redisson</artifactId>
+            <version>3.16.8</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>io.netty</groupId>
+                    <artifactId>netty-transport</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>io.netty</groupId>
+                    <artifactId>netty-resolver</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>io.netty</groupId>
+                    <artifactId>netty-handler</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>io.netty</groupId>
+                    <artifactId>netty-codec</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>io.netty</groupId>
+                    <artifactId>netty-buffer</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>io.netty</groupId>
+                    <artifactId>netty-common</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>io.netty</groupId>
+                    <artifactId>netty-resolver-dns</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>net.bytebuddy</groupId>
+                    <artifactId>byte-buddy</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-api</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
         <!-- TinkerPop -->
         <dependency>
             <groupId>org.apache.tinkerpop</groupId>

janusgraph-core/src/main/java/org/janusgraph/diskstorage/Backend.java

@@ -43,6 +43,7 @@
 import org.janusgraph.diskstorage.keycolumnvalue.StoreTransaction;
 import org.janusgraph.diskstorage.keycolumnvalue.cache.CacheTransaction;
 import org.janusgraph.diskstorage.keycolumnvalue.cache.ExpirationKCVSCache;
+import org.janusgraph.diskstorage.keycolumnvalue.cache.ExpirationKCVSRedisCache;
 import org.janusgraph.diskstorage.keycolumnvalue.cache.KCVSCache;
 import org.janusgraph.diskstorage.keycolumnvalue.cache.NoKCVSCache;
 import org.janusgraph.diskstorage.keycolumnvalue.keyvalue.OrderedKeyValueStoreManager;
@@ -86,6 +87,7 @@
 import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.ALLOW_CUSTOM_VERTEX_ID_TYPES;
 import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.BASIC_METRICS;
 import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.BUFFER_SIZE;
+import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.CACHE_TYPE;
 import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.DB_CACHE;
 import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.DB_CACHE_CLEAN_WAIT;
 import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.DB_CACHE_SIZE;
@@ -156,6 +158,7 @@
 
     public static final String SYSTEM_TX_LOG_NAME = "txlog";
     public static final String SYSTEM_MGMT_LOG_NAME = "systemlog";
+    public static final String REDIS_TAG = "redis";
 
     public static final double EDGESTORE_CACHE_PERCENT = 0.8;
     public static final double INDEXSTORE_CACHE_PERCENT = 0.2;
@@ -362,8 +365,22 @@
                 long edgeStoreCacheSize = Math.round(cacheSizeBytes * EDGESTORE_CACHE_PERCENT);
                 long indexStoreCacheSize = Math.round(cacheSizeBytes * INDEXSTORE_CACHE_PERCENT);
 
-                edgeStore = new ExpirationKCVSCache(edgeStoreRaw,getMetricsCacheName(EDGESTORE_NAME),expirationTime,cleanWaitTime,edgeStoreCacheSize);
-                indexStore = new ExpirationKCVSCache(indexStoreRaw,getMetricsCacheName(INDEXSTORE_NAME),expirationTime,cleanWaitTime,indexStoreCacheSize);
+                String cacheType = configuration.get(CACHE_TYPE);
+
+                if(REDIS_TAG.equals(cacheType)){
+                    log.info("======== Configuring redis cache ========");
+                    edgeStore = new ExpirationKCVSRedisCache(edgeStoreRaw,getMetricsCacheName(EDGESTORE_NAME)!=null?getMetricsCacheName(EDGESTORE_NAME)
+                        :"edgeStore",expirationTime,cleanWaitTime,
+                        edgeStoreCacheSize, configuration);
+                    indexStore = new ExpirationKCVSRedisCache(indexStoreRaw,getMetricsCacheName(INDEXSTORE_NAME)!=null?
+                        getMetricsCacheName(INDEXSTORE_NAME):"indexStore",expirationTime,cleanWaitTime,
+                        indexStoreCacheSize, configuration);
+                }else{
+                    log.info("======== Configuring inmemory cache ========");
+                    edgeStore = new ExpirationKCVSCache(edgeStoreRaw,getMetricsCacheName(EDGESTORE_NAME),expirationTime,cleanWaitTime,edgeStoreCacheSize);
+                    indexStore = new ExpirationKCVSCache(indexStoreRaw,getMetricsCacheName(INDEXSTORE_NAME),expirationTime,cleanWaitTime,indexStoreCacheSize);
+                }
+
             } else {
                 edgeStore = new NoKCVSCache(edgeStoreRaw);
                 indexStore = new NoKCVSCache(indexStoreRaw);

janusgraph-core/src/main/java/org/janusgraph/diskstorage/EntryList.java

@@ -14,6 +14,7 @@
 
 package org.janusgraph.diskstorage;
 
+import java.io.Serializable;
 import java.util.AbstractList;
 import java.util.Iterator;
 import java.util.List;
@@ -45,7 +46,7 @@ public interface EntryList extends List<Entry> {
 
     EmptyList EMPTY_LIST = new EmptyList();
 
-    class EmptyList extends AbstractList<Entry> implements EntryList {
+    class EmptyList extends AbstractList<Entry> implements EntryList, Serializable {
 
         @Override
         public Entry get(int index) {

janusgraph-core/src/main/java/org/janusgraph/diskstorage/keycolumnvalue/KeySliceQuery.java

@@ -17,14 +17,15 @@
 import com.google.common.base.Preconditions;
 import org.janusgraph.diskstorage.StaticBuffer;
 
+import java.io.Serializable;
 import java.util.Objects;
 
 /**
  * Extends {@link SliceQuery} by a key that identifies the location of the slice in the key-ring.
  * @author Matthias Broecheler (me@matthiasb.com)
  */
 
-public class KeySliceQuery extends SliceQuery {
+public class KeySliceQuery extends SliceQuery implements Serializable {
 
     private final StaticBuffer key;
 

janusgraph-core/src/main/java/org/janusgraph/diskstorage/keycolumnvalue/SliceQuery.java

@@ -23,6 +23,7 @@
 import org.janusgraph.graphdb.query.BackendQuery;
 import org.janusgraph.graphdb.query.BaseQuery;
 
+import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -37,7 +38,7 @@
  * @author Matthias Broecheler (me@matthiasb.com)
  */
 
-public class SliceQuery extends BaseQuery implements BackendQuery<SliceQuery> {
+public class SliceQuery extends BaseQuery implements BackendQuery<SliceQuery>, Serializable {
 
     private final StaticBuffer sliceStart;
     private final StaticBuffer sliceEnd;