IGNITE-8560 Update index validation utility to use statistical check …

…approach - Fixes apache#4051.

Signed-off-by: Ivan Rakov <irakov@apache.org>

(cherry-picked from commit #76e1fe754d7a8bd059d7ab64d17cbefa4913a702)

modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java

@@ -181,6 +181,12 @@ public class CommandHandler {
     /** */
     private static final String BASELINE_SET_VERSION = "version";
 
+    /** Parameter name for validate_indexes command. */
+    static final String VI_CHECK_FIRST = "checkFirst";
+
+    /** Parameter name for validate_indexes command. */
+    static final String VI_CHECK_THROUGH = "checkThrough";
+
     /** */
     static final String WAL_PRINT = "print";
 
@@ -583,10 +589,12 @@ private void printCacheHelp() {
         usage("  Show information about caches, groups or sequences that match a regex:", CACHE, " list regexPattern [groups|seq] [nodeId]");
         usage("  Show hot keys that are point of contention for multiple transactions:", CACHE, " contention minQueueSize [nodeId] [maxPrint]");
         usage("  Verify partition counters and hashes between primary and backups on idle cluster:", CACHE, " idle_verify [cache1,...,cacheN]");
-        usage("  Validate custom indexes on idle cluster:", CACHE, " validate_indexes [cache1,...,cacheN] [nodeId]");
+        usage("  Validate custom indexes on idle cluster:", CACHE, " validate_indexes [cache1,...,cacheN] [nodeId] [checkFirst|checkThrough]");
 
-        log("  If [nodeId] is not specified, cont and validate_indexes commands will be broadcasted to all server nodes.");
+        log("  If [nodeId] is not specified, contention and validate_indexes commands will be broadcasted to all server nodes.");
         log("  Another commands where [nodeId] is optional will run on a random server node.");
+        log("  checkFirst numeric parameter for validate_indexes specifies number of first K keys to be validated.");
+        log("  checkThrough numeric parameter for validate_indexes allows to check each Kth key.");
         nl();
     }
 
@@ -624,7 +632,11 @@ private void cacheContention(GridClient client, CacheArguments cacheArgs) throws
      * @param cacheArgs Cache args.
      */
     private void cacheValidateIndexes(GridClient client, CacheArguments cacheArgs) throws GridClientException {
-        VisorValidateIndexesTaskArg taskArg = new VisorValidateIndexesTaskArg(cacheArgs.caches());
+        VisorValidateIndexesTaskArg taskArg = new VisorValidateIndexesTaskArg(
+            cacheArgs.caches(),
+            cacheArgs.checkFirst(),
+            cacheArgs.checkThrough()
+        );
 
         UUID nodeId = cacheArgs.nodeId() == null ? BROADCAST_UUID : cacheArgs.nodeId();
 
@@ -1407,7 +1419,8 @@ private CacheArguments parseAndValidateCacheArgs() {
                 break;
 
             case IDLE_VERIFY:
-                parseCacheNamesIfPresent(cacheArgs);
+                if (hasNextCacheArg())
+                    parseCacheNames(nextArg(""), cacheArgs);
 
                 break;
 
@@ -1425,10 +1438,53 @@ private CacheArguments parseAndValidateCacheArgs() {
                 break;
 
             case VALIDATE_INDEXES:
-                parseCacheNamesIfPresent(cacheArgs);
+                int argsCnt = 0;
 
-                if (hasNextCacheArg())
-                    cacheArgs.nodeId(UUID.fromString(nextArg("")));
+                while (hasNextCacheArg() && argsCnt++ < 4) {
+                    String arg = nextArg("");
+
+                    if (VI_CHECK_FIRST.equals(arg) || VI_CHECK_THROUGH.equals(arg)) {
+                        if (!hasNextCacheArg())
+                            throw new IllegalArgumentException("Numeric value for '" + arg + "' parameter expected.");
+
+                        int numVal;
+
+                        String numStr = nextArg("");
+
+                        try {
+                            numVal = Integer.parseInt(numStr);
+                        }
+                        catch (IllegalArgumentException e) {
+                            throw new IllegalArgumentException(
+                                "Not numeric value was passed for '"
+                                    + arg
+                                    + "' parameter: "
+                                    + numStr
+                            );
+                        }
+
+                        if (numVal <= 0)
+                            throw new IllegalArgumentException("Value for '" + arg + "' property should be positive.");
+
+                        if (VI_CHECK_FIRST.equals(arg))
+                            cacheArgs.checkFirst(numVal);
+                        else
+                            cacheArgs.checkThrough(numVal);
+
+                        continue;
+                    }
+
+                    try {
+                        cacheArgs.nodeId(UUID.fromString(arg));
+
+                        continue;
+                    }
+                    catch (IllegalArgumentException ignored) {
+                        //No-op.
+                    }
+
+                    parseCacheNames(arg, cacheArgs);
+                }
 
                 break;
 
@@ -1473,22 +1529,18 @@ private boolean hasNextCacheArg() {
     /**
      * @param cacheArgs Cache args.
      */
-    private void parseCacheNamesIfPresent(CacheArguments cacheArgs) {
-        if (hasNextCacheArg()) {
-            String cacheNames = nextArg("");
-
-            String[] cacheNamesArr = cacheNames.split(",");
-            Set<String> cacheNamesSet = new HashSet<>();
+    private void parseCacheNames(String cacheNames, CacheArguments cacheArgs) {
+        String[] cacheNamesArr = cacheNames.split(",");
+        Set<String> cacheNamesSet = new HashSet<>();
 
-            for (String cacheName : cacheNamesArr) {
-                if (F.isEmpty(cacheName))
-                    throw new IllegalArgumentException("Non-empty cache names expected.");
+        for (String cacheName : cacheNamesArr) {
+            if (F.isEmpty(cacheName))
+                throw new IllegalArgumentException("Non-empty cache names expected.");
 
-                cacheNamesSet.add(cacheName.trim());
-            }
-
-            cacheArgs.caches(cacheNamesSet);
+            cacheNamesSet.add(cacheName.trim());
         }
+
+        cacheArgs.caches(cacheNamesSet);
     }
 
     /**

modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java

@@ -46,6 +46,12 @@ public class CacheArguments {
     /** Max print. */
     private int maxPrint;
 
+    /** validate_indexes 'checkFirst' argument */
+    private int checkFirst = -1;
+
+    /** validate_indexes 'checkThrough' argument */
+    private int checkThrough = -1;
+
     /** Cache view command. */
     private @Nullable VisorViewCacheCmd cacheCmd;
 
@@ -160,4 +166,32 @@ public int maxPrint() {
     public void maxPrint(int maxPrint) {
         this.maxPrint = maxPrint;
     }
+
+    /**
+     *  @return Max number of entries to be checked.
+     */
+    public int checkFirst() {
+        return checkFirst;
+    }
+
+    /**
+     * @param checkFirst Max number of entries to be checked.
+     */
+    public void checkFirst(int checkFirst) {
+        this.checkFirst = checkFirst;
+    }
+
+    /**
+     * @return Number of entries to check through.
+     */
+    public int checkThrough() {
+        return checkThrough;
+    }
+
+    /**
+     * @param checkThrough Number of entries to check through.
+     */
+    public void checkThrough(int checkThrough) {
+        this.checkThrough = checkThrough;
+    }
 }

modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTaskArg.java

@@ -35,6 +35,12 @@ public class VisorValidateIndexesTaskArg extends VisorDataTransferObject {
     /** Caches. */
     private Set<String> caches;
 
+    /** Check first K elements. */
+    private int checkFirst;
+
+    /** Check through K element (skip K-1, check Kth). */
+    private int checkThrough;
+
     /**
      * Default constructor.
      */
@@ -45,8 +51,10 @@ public VisorValidateIndexesTaskArg() {
     /**
      * @param caches Caches.
      */
-    public VisorValidateIndexesTaskArg(Set<String> caches) {
+    public VisorValidateIndexesTaskArg(Set<String> caches, int checkFirst, int checkThrough) {
         this.caches = caches;
+        this.checkFirst = checkFirst;
+        this.checkThrough = checkThrough;
     }
 
 
@@ -57,14 +65,44 @@ public Set<String> getCaches() {
         return caches;
     }
 
+    /**
+     * @return checkFirst.
+     */
+    public int getCheckFirst() {
+        return checkFirst;
+    }
+
+    /**
+     * @return checkThrough.
+     */
+    public int getCheckThrough() {
+        return checkThrough;
+    }
+
     /** {@inheritDoc} */
     @Override protected void writeExternalData(ObjectOutput out) throws IOException {
         U.writeCollection(out, caches);
+        out.writeInt(checkFirst);
+        out.writeInt(checkThrough);
     }
 
     /** {@inheritDoc} */
     @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
         caches = U.readSet(in);
+
+        if (protoVer > V1) {
+            checkFirst = in.readInt();
+            checkThrough = in.readInt();
+        }
+        else {
+            checkFirst = -1;
+            checkThrough = -1;
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public byte getProtocolVersion() {
+        return V2;
     }
 
     /** {@inheritDoc} */

modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java

@@ -21,15 +21,20 @@
 import java.util.Collections;
 import java.util.UUID;
 import junit.framework.TestCase;
+import org.apache.ignite.internal.commandline.cache.CacheArguments;
+import org.apache.ignite.internal.commandline.cache.CacheCommand;
 import org.apache.ignite.internal.visor.tx.VisorTxProjection;
 import org.apache.ignite.internal.visor.tx.VisorTxSortOrder;
 import org.apache.ignite.internal.visor.tx.VisorTxTaskArg;
 
 import static java.util.Arrays.asList;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_ENABLE_EXPERIMENTAL_COMMAND;
+import static org.apache.ignite.internal.commandline.Command.CACHE;
 import static org.apache.ignite.internal.commandline.Command.WAL;
 import static org.apache.ignite.internal.commandline.CommandHandler.DFLT_HOST;
 import static org.apache.ignite.internal.commandline.CommandHandler.DFLT_PORT;
+import static org.apache.ignite.internal.commandline.CommandHandler.VI_CHECK_FIRST;
+import static org.apache.ignite.internal.commandline.CommandHandler.VI_CHECK_THROUGH;
 import static org.apache.ignite.internal.commandline.CommandHandler.WAL_DELETE;
 import static org.apache.ignite.internal.commandline.CommandHandler.WAL_PRINT;
 
@@ -51,6 +56,88 @@ public class CommandHandlerParsingTest extends TestCase {
         super.tearDown();
     }
 
+    /**
+     * validate_indexes command arguments parsing and validation
+     */
+    public void testValidateIndexArguments() {
+        CommandHandler hnd = new CommandHandler();
+
+        //happy case for all parameters
+        try {
+            int expectedCheckFirst = 10;
+            int expectedCheckThrough = 11;
+            UUID nodeId = UUID.randomUUID();
+
+            CacheArguments args = hnd.parseAndValidate(
+                Arrays.asList(
+                    CACHE.text(),
+                    CacheCommand.VALIDATE_INDEXES.text(),
+                    "cache1, cache2",
+                    nodeId.toString(),
+                    VI_CHECK_FIRST,
+                    Integer.toString(expectedCheckFirst),
+                    VI_CHECK_THROUGH,
+                    Integer.toString(expectedCheckThrough)
+                )
+            ).cacheArgs();
+
+            assertEquals("nodeId parameter unexpected value", nodeId, args.nodeId());
+            assertEquals("checkFirst parameter unexpected value", expectedCheckFirst, args.checkFirst());
+            assertEquals("checkThrough parameter unexpected value", expectedCheckThrough, args.checkThrough());
+        }
+        catch (IllegalArgumentException e) {
+            fail("Unexpected exception: " + e);
+        }
+
+        try {
+            int expectedParam = 11;
+            UUID nodeId = UUID.randomUUID();
+
+            CacheArguments args = hnd.parseAndValidate(
+                Arrays.asList(
+                    CACHE.text(),
+                    CacheCommand.VALIDATE_INDEXES.text(),
+                    nodeId.toString(),
+                    VI_CHECK_THROUGH,
+                    Integer.toString(expectedParam)
+                )
+            ).cacheArgs();
+
+            assertNull("caches weren't specified, null value expected", args.caches());
+            assertEquals("nodeId parameter unexpected value", nodeId, args.nodeId());
+            assertEquals("checkFirst parameter unexpected value", -1, args.checkFirst());
+            assertEquals("checkThrough parameter unexpected value", expectedParam, args.checkThrough());
+        }
+        catch (IllegalArgumentException e) {
+            e.printStackTrace();
+        }
+
+        try {
+            hnd.parseAndValidate(
+                Arrays.asList(
+                    CACHE.text(),
+                    CacheCommand.VALIDATE_INDEXES.text(),
+                    VI_CHECK_FIRST,
+                    "0"
+                )
+            );
+
+            fail("Expected exception hasn't been thrown");
+        }
+        catch (IllegalArgumentException e) {
+            e.printStackTrace();
+        }
+
+        try {
+            hnd.parseAndValidate(Arrays.asList(CACHE.text(), CacheCommand.VALIDATE_INDEXES.text(), VI_CHECK_THROUGH));
+
+            fail("Expected exception hasn't been thrown");
+        }
+        catch (IllegalArgumentException e) {
+            e.printStackTrace();
+        }
+    }
+
     /**
      * Test that experimental command (i.e. WAL command) is disabled by default.
      */