Skip to content
Sign up
This repository

public / voldemort Octocat-spinner-32

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

Updated ConsistencyCheck.java for new feature

  • Loading branch information...
commit efa07700d66c63a2db3c0e799d541288f7cc9e20 1 parent 388cca1
Zhongjie Wu authored

Showing 1 changed file with 254 additions and 106 deletions. Show Diff Stats Hide Diff Stats

  1. 360  src/java/voldemort/utils/ConsistencyCheck.java
360  src/java/voldemort/utils/ConsistencyCheck.java
View 
@@ -17,6 +17,7 @@
17 17 
 import voldemort.client.protocol.admin.AdminClientConfig;
18 18 
 import voldemort.cluster.Cluster;
19 19 
 import voldemort.cluster.Node;
  20 
+import voldemort.cluster.Zone;
20 21 
 import voldemort.routing.RoutingStrategyFactory;
21 22 
 import voldemort.store.StoreDefinition;
22 23 
 import voldemort.versioning.Occurred;
 
@@ -26,6 +27,46 @@
26 27
27 28 
 public class ConsistencyCheck {
28 29
  30 
+    private enum ConsistencyLevel {
  31 
+        FULL,
  32 
+        GREY,
  33 
+        ORANGE,
  34 
+        ANTI_DR
  35 
+    }
  36 
+
  37 
+    private static class ConsistencyCheckStats {
  38 
+
  39 
+        private long consistentKeys;
  40 
+        private long totalKeys;
  41 
+
  42 
+        public ConsistencyCheckStats() {
  43 
+            consistentKeys = 0;
  44 
+            totalKeys = 0;
  45 
+        }
  46 
+
  47 
+        public void setConsistentKeys(long count) {
  48 
+            consistentKeys += count;
  49 
+        }
  50 
+
  51 
+        public void setTotalKeys(long count) {
  52 
+            totalKeys += count;
  53 
+        }
  54 
+
  55 
+        public long getConsistentKeys() {
  56 
+            return consistentKeys;
  57 
+        }
  58 
+
  59 
+        public long getTotalKeys() {
  60 
+            return totalKeys;
  61 
+        }
  62 
+
  63 
+        public void append(ConsistencyCheckStats that) {
  64 
+            consistentKeys += that.getConsistentKeys();
  65 
+            totalKeys += that.getTotalKeys();
  66 
+        }
  67 
+    }
  68 
+
  69 
+    @SuppressWarnings("unchecked")
29 70 
     public static void main(String[] args) throws Exception {
30 71 
         /* parse options */
31 72 
         OptionParser parser = new OptionParser();
 
@@ -34,32 +75,94 @@ public static void main(String[] args) throws Exception {
34 75 
               .withRequiredArg()
35 76 
               .describedAs("bootstrap-url")
36 77 
               .ofType(String.class);
37   
-        parser.accepts("partition", "partition-id")
  78 
+        parser.accepts("partitions", "partition-id")
38 79 
               .withRequiredArg()
39 80 
               .describedAs("partition-id")
  81 
+              .withValuesSeparatedBy(',')
40 82 
               .ofType(Integer.class);
41 83 
         parser.accepts("store", "store name")
42 84 
               .withRequiredArg()
43 85 
               .describedAs("store-name")
44 86 
               .ofType(String.class);
  87 
+        parser.accepts("primary-zone", "primary zone id").withRequiredArg().ofType(Integer.class);
  88 
+        parser.accepts("verbose", "verbose");
45 89 
         OptionSet options = parser.parse(args);
46 90
47 91 
         /* validate options */
  92 
+        boolean verbose = false;
48 93 
         if(options.hasArgument("help")) {
49 94 
             printUsage();
  95 
+            return;
50 96 
         }
51   
-        if(!options.hasArgument("url") || !options.hasArgument("partition")
52   
-           || !options.hasArgument("store")) {
  97 
+        if(!options.hasArgument("url") || !options.hasArgument("partitions")
  98 
+           || !options.hasArgument("store") || !options.hasArgument("primary-zone")) {
53 99 
             printUsage();
  100 
+            return;
  101 
+        }
  102 
+        if(options.has("verbose")) {
  103 
+            verbose = true;
54 104 
         }
55 105
56 106 
         String url = (String) options.valueOf("url");
57 107 
         String storeName = (String) options.valueOf("store");
58   
-        Integer partitionId = (Integer) options.valueOf("partition");
  108 
+        List<Integer> partitionIds = (List<Integer>) options.valuesOf("partitions");
  109 
+        Integer primaryZoneId = (Integer) options.valueOf("primary-zone");
  110 
+
  111 
+        ConsistencyCheckStats globalStats = new ConsistencyCheckStats();
  112 
+        Map<Integer, ConsistencyCheckStats> partitionStatsMap = new HashMap<Integer, ConsistencyCheckStats>();
  113 
+        for(Integer partitionId: partitionIds) {
  114 
+            ConsistencyCheckStats partitionStats = doConsistencyCheck(storeName,
  115 
+                                                                      partitionId,
  116 
+                                                                      url,
  117 
+                                                                      primaryZoneId,
  118 
+                                                                      verbose);
  119 
+            partitionStatsMap.put(partitionId, partitionStats);
  120 
+            globalStats.append(partitionStats);
  121 
+        }
  122 
+
  123 
+        /* print stats */
  124 
+        // partition based
  125 
+        StringBuilder statsString = new StringBuilder();
  126 
+        // each partition
  127 
+        statsString.append("TYPE,Store,ParitionId,KeysConsistent,KeysTotal,Consistency\n");
  128 
+        for(Map.Entry<Integer, ConsistencyCheckStats> entry: partitionStatsMap.entrySet()) {
  129 
+            Integer partitionId = entry.getKey();
  130 
+            ConsistencyCheckStats partitionStats = entry.getValue();
  131 
+            statsString.append("STATS,");
  132 
+            statsString.append(storeName + ",");
  133 
+            statsString.append(partitionId + ",");
  134 
+            statsString.append(partitionStats.getConsistentKeys() + ",");
  135 
+            statsString.append(partitionStats.getTotalKeys() + ",");
  136 
+            statsString.append((double) (partitionStats.getConsistentKeys())
  137 
+                               / (double) partitionStats.getTotalKeys());
  138 
+            statsString.append("\n");
  139 
+        }
  140 
+        // all partitions
  141 
+        statsString.append("STATS,");
  142 
+        statsString.append(storeName + ",");
  143 
+        statsString.append("aggregate,");
  144 
+        statsString.append(globalStats.getConsistentKeys() + ",");
  145 
+        statsString.append(globalStats.getTotalKeys() + ",");
  146 
+        statsString.append((double) (globalStats.getConsistentKeys())
  147 
+                           / (double) globalStats.getTotalKeys());
  148 
+        statsString.append("\n");
  149 
+
  150 
+        System.out.println();
  151 
+        System.out.println(statsString.toString());
  152 
+    }
  153 
+
  154 
+    public static ConsistencyCheckStats doConsistencyCheck(String storeName,
  155 
+                                                           Integer partitionId,
  156 
+                                                           String url,
  157 
+                                                           Integer primaryZoneId,
  158 
+                                                           boolean verbose) throws Exception {
59 159 
         List<Integer> singlePartition = new ArrayList<Integer>();
60 160 
         singlePartition.add(partitionId);
61 161
62 162 
         /* connect to cluster */
  163 
+        if(verbose) {
  164 
+            System.out.println("Connecting to bootstrap server: " + url);
  165 
+        }
63 166 
         AdminClient adminClient = new AdminClient(url, new AdminClientConfig(), 0);
64 167 
         Cluster cluster = adminClient.getAdminClientCluster();
65 168 
 
@@ -118,14 +221,18 @@ public static void main(String[] args) throws Exception {
118 221 
         }
119 222
120 223 
         /* print config info */
121   
-        StringBuilder configInfo = new StringBuilder();
122   
-        configInfo.append("Configuration\n");
123   
-        configInfo.append("=============\n");
124   
-        configInfo.append("  URL: " + url + "\n");
125   
-        configInfo.append("Store: " + storeName + "\n");
126   
-        configInfo.append("ParId: " + partitionId + "\n");
127   
-        configInfo.append("Nodes: " + nodeIdList.toString() + "\n");
128   
-        System.out.println(configInfo);
  224 
+        if(verbose) {
  225 
+            StringBuilder configInfo = new StringBuilder();
  226 
+            configInfo.append("TYPE,Store,PartitionId,Node,ZoneId\n");
  227 
+            for(Integer nodeId: nodeIdList) {
  228 
+                configInfo.append("CONFIG,");
  229 
+                configInfo.append(storeName + ",");
  230 
+                configInfo.append(partitionId + ",");
  231 
+                configInfo.append(nodeId + ",");
  232 
+                configInfo.append(cluster.getNodeById(nodeId).getZoneId() + "\n");
  233 
+            }
  234 
+            System.out.println(configInfo);
  235 
+        }
129 236
130 237 
         /* get entry Iterator from each node */
131 238 
         Map<Integer, Iterator<Pair<ByteArray, Versioned<byte[]>>>> nodeEntriesMap;
 
@@ -164,34 +271,36 @@ public static void main(String[] args) throws Exception {
164 271 
                     ByteArray key = nodeEntry.getFirst();
165 272 
                     Version version = nodeEntry.getSecond().getVersion();
166 273
167   
-                    // try sweep last key fetched by this iterator
168   
-                    if(lastFetchedKey.containsKey(nodeEntries)) {
169   
-                        ByteArray lastKey = lastFetchedKey.get(nodeEntries);
170   
-                        if(key != lastKey) {
171   
-                            if(!fullyFetchedKeys.containsKey(lastKey)) {
172   
-                                fullyFetchedKeys.put(lastKey,
173   
-                                                     new HashSet<Iterator<Pair<ByteArray, Versioned<byte[]>>>>());
174   
-                            }
175   
-                            Set<Iterator<Pair<ByteArray, Versioned<byte[]>>>> lastKeyIterSet = fullyFetchedKeys.get(lastKey);
176   
-                            lastKeyIterSet.add(nodeEntries);
177   
-                            // sweep if fully fetched by all iterators
178   
-                            if(lastKeyIterSet.size() == nodeIdList.size()) {
179   
-                                // keyFetchComplete
180   
-                                if(isConsistent(keyVersionNodeSetMap.get(lastKey),
181   
-                                                storeDefinition.getReplicationFactor())) {
182   
-                                    keyVersionNodeSetMap.remove(lastKey);
183   
-                                    preQualifiedKeys++;
184   
-                                }
185   
-                                fullyFetchedKeys.remove(lastKey);
186   
-                            }
187   
-                        }
188   
-                    }
189   
-                    lastFetchedKey.put(nodeEntries, key);
190   
-
191 274 
                     if(retentionChecker.isExpired(version)) {
192 275 
                         expiredRecords++;
193 276 
                         continue;
194 277 
                     } else {
  278 
+                        // try sweep last key fetched by this iterator
  279 
+                        if(lastFetchedKey.containsKey(nodeEntries)) {
  280 
+                            ByteArray lastKey = lastFetchedKey.get(nodeEntries);
  281 
+                            if(key != lastKey) {
  282 
+                                if(!fullyFetchedKeys.containsKey(lastKey)) {
  283 
+                                    fullyFetchedKeys.put(lastKey,
  284 
+                                                         new HashSet<Iterator<Pair<ByteArray, Versioned<byte[]>>>>());
  285 
+                                }
  286 
+                                Set<Iterator<Pair<ByteArray, Versioned<byte[]>>>> lastKeyIterSet = fullyFetchedKeys.get(lastKey);
  287 
+                                lastKeyIterSet.add(nodeEntries);
  288 
+                                // sweep if fully fetched by all iterators
  289 
+                                if(lastKeyIterSet.size() == nodeIdList.size()) {
  290 
+                                    // keyFetchComplete
  291 
+                                    ConsistencyLevel level = determineConsistency(keyVersionNodeSetMap.get(lastKey),
  292 
+                                                                                  storeDefinition,
  293 
+                                                                                  cluster,
  294 
+                                                                                  primaryZoneId);
  295 
+                                    if(level == ConsistencyLevel.FULL) {
  296 
+                                        keyVersionNodeSetMap.remove(lastKey);
  297 
+                                        preQualifiedKeys++;
  298 
+                                    }
  299 
+                                    fullyFetchedKeys.remove(lastKey);
  300 
+                                }
  301 
+                            }
  302 
+                        }
  303 
+                        lastFetchedKey.put(nodeEntries, key);
195 304 
                         // initialize key -> Map<Version, Set<nodeId>>
196 305 
                         if(!keyVersionNodeSetMap.containsKey(key)) {
197 306 
                             keyVersionNodeSetMap.put(key, new HashMap<Version, Set<Integer>>());
 
@@ -229,12 +338,11 @@ public static void main(String[] args) throws Exception {
229 338 
                 }
230 339 
             }
231 340 
             // stats reporting
232   
-            if(System.currentTimeMillis() > lastReportTimeMs + reportPeriodMs) {
  341 
+            if(verbose && System.currentTimeMillis() > lastReportTimeMs + reportPeriodMs) {
233 342 
                 long currentTimeMs = System.currentTimeMillis();
234 343 
                 System.out.println("Progress Report");
235 344 
                 System.out.println("===============");
236 345 
                 System.out.println("    Number of records Scanned: " + numRecordsScanned);
237   
-                System.out.println("  Total number of unique keys: " + keyVersionNodeSetMap.size());
238 346 
                 System.out.println("   Records Ignored(Retention): " + expiredRecords);
239 347 
                 System.out.println("Recent fetch speed(records/s): "
240 348 
                                    + (numRecordsScanned - numRecordsScannedLast)
 
@@ -246,91 +354,131 @@ public static void main(String[] args) throws Exception {
246 354 
         } while(anyNodeHasNext);
247 355
248 356 
         // analyzing
249   
-        System.out.println("Analyzing....");
  357 
+        if(verbose) {
  358 
+            System.out.println("Analyzing....");
  359 
+        }
250 360 
         cleanIneligibleKeys(keyVersionNodeSetMap, storeDefinition.getRequiredWrites());
  361 
+
  362 
+        // clean the rest of consistent keys
  363 
+        Set<ByteArray> keysToDelete = new HashSet<ByteArray>();
  364 
+        for(ByteArray key: keyVersionNodeSetMap.keySet()) {
  365 
+            ConsistencyLevel level = determineConsistency(keyVersionNodeSetMap.get(key),
  366 
+                                                          storeDefinition,
  367 
+                                                          cluster,
  368 
+                                                          primaryZoneId);
  369 
+            if(level == ConsistencyLevel.FULL) {
  370 
+                keysToDelete.add(key);
  371 
+            }
  372 
+        }
  373 
+        for(ByteArray key: keysToDelete) {
  374 
+            keyVersionNodeSetMap.remove(key);
  375 
+            preQualifiedKeys++;
  376 
+        }
  377 
+
251 378 
         long totalKeys = keyVersionNodeSetMap.size() + preQualifiedKeys;
252 379 
         long totalKeysConsistent = preQualifiedKeys;
253 380
254   
-        // print some stats
255   
-        StringBuilder totalStats = new StringBuilder();
256   
-        totalStats.append("KeysTotal, KeysConsistent, Consistency\n");
257   
-        totalStats.append(totalKeys + ", " + totalKeysConsistent + ", ");
258   
-        totalStats.append((double) totalKeysConsistent / (double) totalKeys);
259   
-        totalStats.append("\n");
260   
-        System.out.println(totalStats.toString());
261   
-
262   
-        // zone-wise consistency
263   
-        Map<Integer, Integer> zonePartialConsistentKeyCount = new HashMap<Integer, Integer>();
264   
-        for(Map.Entry<Integer, Set<Integer>> zoneToNodeSetEntry: zoneToNodeIds.entrySet()) {
265   
-            Integer zoneId = zoneToNodeSetEntry.getKey();
266   
-            Set<Integer> zoneNodeSet = zoneToNodeSetEntry.getValue();
267   
-            int partialConsistentKeyCount = 0;
  381 
+        // print inconsistent keys
  382 
+        if(verbose) {
  383 
+            StringBuilder record = new StringBuilder();
  384 
+            record.append("TYPE,Store,ParId,Key,ServerSet,VersionTS,VectorClock,ConsistentyLevel\n");
268 385 
             for(Map.Entry<ByteArray, Map<Version, Set<Integer>>> entry: keyVersionNodeSetMap.entrySet()) {
269   
-                boolean partialConsistent = true;
270   
-                for(Set<Integer> keyVersionNodeSet: entry.getValue().values()) {
271   
-                    partialConsistent = partialConsistent
272   
-                                        && keyVersionNodeSet.containsAll(zoneNodeSet);
273   
-                }
274   
-                if(partialConsistent) {
275   
-                    partialConsistentKeyCount++;
  386 
+                ByteArray key = entry.getKey();
  387 
+                Map<Version, Set<Integer>> versionMap = entry.getValue();
  388 
+                for(Map.Entry<Version, Set<Integer>> versionSet: versionMap.entrySet()) {
  389 
+                    Version version = versionSet.getKey();
  390 
+                    Set<Integer> nodeSet = versionSet.getValue();
  391 
+                    record.append("BAD_KEY,");
  392 
+                    record.append(storeName + ",");
  393 
+                    record.append(partitionId + ",");
  394 
+                    record.append(ByteUtils.toHexString(key.get()) + ",");
  395 
+                    record.append(nodeSet.toString().replaceAll(", ", ";") + ",");
  396 
+                    record.append(((VectorClock) version).getTimestamp() + ",");
  397 
+                    record.append(version.toString()
  398 
+                                         .replaceAll(", ", ";")
  399 
+                                         .replaceAll(" ts:[0-9]*", "")
  400 
+                                         .replaceAll("version\\((.*)\\)", "[$1]")
  401 
+                                  + ",");
  402 
+                    record.append(determineConsistency(versionMap,
  403 
+                                                       storeDefinition,
  404 
+                                                       cluster,
  405 
+                                                       primaryZoneId).toString()
  406 
+                                  + "\n");
276 407 
                 }
277 408 
             }
278   
-            zonePartialConsistentKeyCount.put(zoneId, partialConsistentKeyCount);
  409 
+            System.out.println(record.toString());
279 410 
         }
280 411
281   
-        // print zone-wise consistency result
282   
-        System.out.println("storeName,zoneId,consistency");
283   
-        for(Integer zoneId: zoneToNodeIds.keySet()) {
284   
-            long zoneConsistentKeys = totalKeysConsistent
285   
-                                      + zonePartialConsistentKeyCount.get(zoneId);
286   
-            System.out.println(storeName + "," + zoneId + "," + (double) (zoneConsistentKeys)
287   
-                               / (double) totalKeys);
288   
-        }
  412 
+        ConsistencyCheckStats stats = new ConsistencyCheckStats();
  413 
+        stats.setConsistentKeys(totalKeysConsistent);
  414 
+        stats.setTotalKeys(totalKeys);
  415 
+
  416 
+        return stats;
289 417 
     }
290 418
291 419 
     public static void printUsage() {
292   
-        System.out.println("Usage: \n--partition <partitionId> --url <url> --store <storeName>");
  420 
+        System.out.println("Usage: \n--partitions <partitionId,partitionId..> --url <url> --store <storeName> --primary-zone <primary-zone-id>");
293 421 
     }
294 422
295   
-    // public static Map<Integer, Integer> cleanConsistentKeys(Map<ByteArray,
296   
-    // Map<Version, Set<Integer>>> keyVersionNodeSetMap,
297   
-    // int replicationFactor) {
298   
-    // Set<ByteArray> keysToDelete = new HashSet<ByteArray>();
299   
-    // Map<Integer, Integer> consistentVersionsCountMap = new HashMap<Integer,
300   
-    // Integer>();
301   
-    // for(Map.Entry<ByteArray, Map<Version, Set<Integer>>> entry:
302   
-    // keyVersionNodeSetMap.entrySet()) {
303   
-    // ByteArray key = entry.getKey();
304   
-    // Map<Version, Set<Integer>> versionNodeSetMap = entry.getValue();
305   
-    // Integer versionCount = versionNodeSetMap.size();
306   
-    // boolean consistent = isConsistent(versionNodeSetMap, replicationFactor);
307   
-    // if(consistent) {
308   
-    // if(consistentVersionsCountMap.containsKey(versionCount)) {
309   
-    // Integer count = consistentVersionsCountMap.get(versionCount);
310   
-    // consistentVersionsCountMap.put(versionCount, new Integer(count + 1));
311   
-    // } else {
312   
-    // consistentVersionsCountMap.put(versionCount, new Integer(1));
313   
-    // }
314   
-    // keyVersionNodeSetMap.put(key, null);
315   
-    // keysToDelete.add(key);
316   
-    // }
317   
-    // }
318   
-    // for(ByteArray deleteThisKey: keysToDelete) {
319   
-    // keyVersionNodeSetMap.remove(deleteThisKey);
320   
-    // }
321   
-    // keysToDelete = null;
322   
-    // System.gc();
323   
-    // return consistentVersionsCountMap;
324   
-    // }
325   
-
326   
-    public static boolean isConsistent(Map<Version, Set<Integer>> versionNodeSetMap,
327   
-                                       int replicationFactor) {
328   
-        boolean consistent = true;
  423 
+    public static ConsistencyLevel determineConsistency(Map<Version, Set<Integer>> versionNodeSetMap,
  424 
+                                                        StoreDefinition storeDef,
  425 
+                                                        Cluster cluster,
  426 
+                                                        int primaryZoneId) {
  427 
+        boolean fullyConsistent = true;
  428 
+        Version latestVersion = null;
329 429 
         for(Map.Entry<Version, Set<Integer>> versionNodeSetEntry: versionNodeSetMap.entrySet()) {
  430 
+            Version version = versionNodeSetEntry.getKey();
  431 
+            if(latestVersion == null
  432 
+               || ((VectorClock) latestVersion).getTimestamp() < ((VectorClock) version).getTimestamp()) {
  433 
+                latestVersion = version;
  434 
+            }
330 435 
             Set<Integer> nodeSet = versionNodeSetEntry.getValue();
331   
-            consistent = consistent && (nodeSet.size() == replicationFactor);
  436 
+            fullyConsistent = fullyConsistent
  437 
+                              && (nodeSet.size() == storeDef.getReplicationFactor());
  438 
+        }
  439 
+        if(fullyConsistent) {
  440 
+            return ConsistencyLevel.FULL;
  441 
+        } else {
  442 
+            Set<Integer> nodeSet = versionNodeSetMap.get(latestVersion);
  443 
+            // latest write consistent, effectively consistent
  444 
+            if(nodeSet.size() == storeDef.getReplicationFactor()) {
  445 
+                return ConsistencyLevel.GREY;
  446 
+            }
  447 
+            // timeout write
  448 
+            if(nodeSet.size() <= storeDef.getRequiredWrites()) {
  449 
+                return ConsistencyLevel.GREY;
  450 
+            }
  451 
+            // DR-category: if other zone does not have as many available nodes
  452 
+            // as primary zone
  453 
+            Map<Integer, Integer> zoneToAvailableNodeCounts = new HashMap<Integer, Integer>();
  454 
+            for(Integer nodeId: nodeSet) {
  455 
+                Integer zoneId = cluster.getNodeById(nodeId).getZoneId();
  456 
+                Integer count = 0;
  457 
+                if(zoneToAvailableNodeCounts.containsKey(zoneId)) {
  458 
+                    count = zoneToAvailableNodeCounts.get(zoneId);
  459 
+                }
  460 
+                count++;
  461 
+                zoneToAvailableNodeCounts.put(zoneId, count);
  462 
+            }
  463 
+            Integer primaryZoneAvailableNodeCounts = zoneToAvailableNodeCounts.get(primaryZoneId);
  464 
+            if(primaryZoneAvailableNodeCounts == null) {
  465 
+                primaryZoneAvailableNodeCounts = 0;
  466 
+            }
  467 
+            for(Zone zone: cluster.getZones()) {
  468 
+                Integer zoneId = zone.getId();
  469 
+                // if not primary zone and has less nodes available than primary
  470 
+                // zone
  471 
+                if(primaryZoneId != zoneId) {
  472 
+                    if(!zoneToAvailableNodeCounts.containsKey(zoneId)
  473 
+                       || zoneToAvailableNodeCounts.get(zoneId) == null
  474 
+                       || zoneToAvailableNodeCounts.get(zoneId) < primaryZoneAvailableNodeCounts) {
  475 
+                        return ConsistencyLevel.ANTI_DR;
  476 
+                    }
  477 
+                }
  478 
+            }
  479 
+            // other intermittent inconsistent state
  480 
+            return ConsistencyLevel.ORANGE;
332 481 
         }
333   
-        return consistent;
334 482 
     }
335 483
336 484 
     public static void cleanIneligibleKeys(Map<ByteArray, Map<Version, Set<Integer>>> keyVersionNodeSetMap,

0 notes on commit efa0770

Please sign in to comment.
Something went wrong with that request. Please try again.