Create herding performance test

[keith-turner]

I wrote the following test while working on apache/accumulo#990. This could be turned into a performance test.

public class HerdTest {

  private static final byte[] E = new byte[] {};
  private static final byte[] FAM = "pinky".getBytes();

  private static final int NUM_ROWS = 1_000_000;
  private static final int NUM_COLS = 10;

  public static void main(String[] args) throws Exception {

    Connector conn = CmdUtil.getConnector();

    if (!conn.tableOperations().exists("herd")) {
      conn.tableOperations().create("herd", new NewTableConfiguration().setProperties(
          Collections.singletonMap(Property.TABLE_BLOCKCACHE_ENABLED.getKey(), "true")));
      write(conn);
      conn.tableOperations().flush("herd", null, null, true);
    }

    testHerd(conn, 32);
  }

  private static void testHerd(Connector conn, int nt)
      throws InterruptedException, ExecutionException {
    ExecutorService tp = Executors.newFixedThreadPool(nt);
    final CyclicBarrier cb = new CyclicBarrier(nt);

    long t1 = System.currentTimeMillis();
    List<Future<?>> futures = new ArrayList<>();
    for (int i = 0; i < nt; i++) {
      Future<?> f = tp.submit(new Runnable() {
        @Override
        public void run() {
          try {
            Scanner scanner = conn.createScanner("herd", Authorizations.EMPTY);

            for (int i = 0; i < 1000; i++) {

              // System.out.println(Thread.currentThread().getId()+" "+i);

              cb.await();

              byte[] row = FastFormat.toZeroPaddedString(i * 1000, 8, 16, E);

              scanner.setRange(Range.exact(new String(row)));
              for (Entry<Key,Value> entry : scanner) {

              }
            }
          } catch (Exception e) {
            e.printStackTrace();
          }
        }
      });
      futures.add(f);
    }

    for (Future<?> future : futures) {
      future.get();
    }

    long t2 = System.currentTimeMillis();

    System.out.println(t2 - t1);

    // scanner.close();
    tp.shutdown();
  }

  private static void write(Connector conn) throws Exception {

    try (BatchWriter bw = conn.createBatchWriter("herd", new BatchWriterConfig())) {
      Random rand = new Random();

      for (int r = 0; r < NUM_ROWS; r++) {
        byte[] row = FastFormat.toZeroPaddedString(r, 8, 16, E);
        Mutation m = new Mutation(row);
        for (int c = 0; c < NUM_COLS; c++) {
          byte[] qual = FastFormat.toZeroPaddedString(c, 4, 16, E);

          byte[] val = new byte[32];
          rand.nextBytes(val);

          m.put(FAM, qual, val);
        }

        bw.addMutation(m);
      }
    }
  }
}

[milleruntime]

I ran some performance test with 32 threads trying to load the same bock at the same time for 1000 times.

This what this test is doing?

[keith-turner]

This what this test is doing?

Yes

[ctubbsii]

This what this test is doing?

Yes

As long as explanatory comments are added when the test is created... 😺

[lbschanno]

I am working on this.