diff --git a/src/test/java/com/yahoo/sketches/benchmark/BenchmarkMain.java b/src/test/java/com/yahoo/sketches/benchmark/BenchmarkMain.java new file mode 100644 index 000000000..08afeaf9e --- /dev/null +++ b/src/test/java/com/yahoo/sketches/benchmark/BenchmarkMain.java @@ -0,0 +1,129 @@ +package com.yahoo.sketches.benchmark; + +import java.util.ArrayList; +import java.util.List; + +/** + */ +public class BenchmarkMain +{ + public static void main(String[] args) + { + List benchmarks = new ArrayList(){{ + this.add(new ThetaBenchmark(13)); + this.add(new ThetaMemoryBenchmark(13)); + }}; + + runBenchmarks(benchmarks, 20, 300, powerLawDistribution); + } + + private static void runBenchmarks( + List benchmarks, + int increment, + int numTimes, + List distribution + ) + { + int numSketches = 0; + for (SketchBenchmark.Spec spec : distribution) { + numSketches += spec.getNumSketches(); + } + + for (SketchBenchmark benchmark : benchmarks) { + System.out.printf("Starting benchmark[%s]%n", benchmark); + long start = System.currentTimeMillis(); + benchmark.setup(numSketches, powerLawDistribution); + System.out.printf("benchmark[%s] setup done in %,d millis.%n", benchmark, System.currentTimeMillis() - start); + start = System.currentTimeMillis(); + benchmark.runNTimes(increment); + System.out.printf("benchmark[%s] priming[%s] done in %,d millis.%n", benchmark, increment, System.currentTimeMillis() - start); + doGC(); + + + for (int i = 0; i < numTimes; i+=increment) { + start = System.currentTimeMillis(); + benchmark.runNTimes(increment); + long time = System.currentTimeMillis() - start; + System.out.printf( + "Benchmark[%s], %,d runs => %,d millis (%,d ms/run), %,d/sec%n", + benchmark, + i + increment, + time, + (int) (time / (double) increment), + (int) ((1000 / (time / (double) increment)) * numSketches) + ); + doGC(); + } + System.out.printf("Done with benchmark[%s]%n", benchmark); + } + } + + private static void doGC() + { + for (int i = 0; i < 10; ++i) { + System.gc(); + } + } + + + public static List powerLawDistribution = new ArrayList(){{ + this.add(new SketchBenchmark.Spec(0, 44129)); + this.add(new SketchBenchmark.Spec(1, 431561)); + this.add(new SketchBenchmark.Spec(2, 129063)); + this.add(new SketchBenchmark.Spec(3, 64821)); + this.add(new SketchBenchmark.Spec(4, 67522)); + this.add(new SketchBenchmark.Spec(6, 20291)); + this.add(new SketchBenchmark.Spec(7, 15767)); + this.add(new SketchBenchmark.Spec(8, 22975)); + this.add(new SketchBenchmark.Spec(11, 22441)); + this.add(new SketchBenchmark.Spec(14, 14531)); + this.add(new SketchBenchmark.Spec(17, 13472)); + this.add(new SketchBenchmark.Spec(22, 13253)); + this.add(new SketchBenchmark.Spec(28, 9002)); + this.add(new SketchBenchmark.Spec(35, 8406)); + this.add(new SketchBenchmark.Spec(45, 7618)); + this.add(new SketchBenchmark.Spec(57, 6349)); + this.add(new SketchBenchmark.Spec(71, 5194)); + this.add(new SketchBenchmark.Spec(89, 4524)); + this.add(new SketchBenchmark.Spec(112, 4032)); + this.add(new SketchBenchmark.Spec(141, 3397)); + this.add(new SketchBenchmark.Spec(178, 2935)); + this.add(new SketchBenchmark.Spec(224, 2516)); + this.add(new SketchBenchmark.Spec(282, 2118)); + this.add(new SketchBenchmark.Spec(355, 1825)); + this.add(new SketchBenchmark.Spec(447, 1527)); + this.add(new SketchBenchmark.Spec(561, 1269)); + this.add(new SketchBenchmark.Spec(709, 1088)); + this.add(new SketchBenchmark.Spec(890, 900)); + this.add(new SketchBenchmark.Spec(1118, 767)); + this.add(new SketchBenchmark.Spec(1410, 654)); + this.add(new SketchBenchmark.Spec(1776, 550)); + this.add(new SketchBenchmark.Spec(2246, 469)); + this.add(new SketchBenchmark.Spec(2813, 353)); + this.add(new SketchBenchmark.Spec(3552, 325)); + this.add(new SketchBenchmark.Spec(4472, 252)); + this.add(new SketchBenchmark.Spec(5639, 249)); + this.add(new SketchBenchmark.Spec(7022, 187)); + this.add(new SketchBenchmark.Spec(8952, 150)); + this.add(new SketchBenchmark.Spec(11270, 138)); + this.add(new SketchBenchmark.Spec(14198, 106)); + this.add(new SketchBenchmark.Spec(17544, 74)); + this.add(new SketchBenchmark.Spec(22145, 81)); + this.add(new SketchBenchmark.Spec(27848, 50)); + this.add(new SketchBenchmark.Spec(35319, 58)); + this.add(new SketchBenchmark.Spec(44267, 33)); + this.add(new SketchBenchmark.Spec(55292, 22)); + this.add(new SketchBenchmark.Spec(72264, 10)); + this.add(new SketchBenchmark.Spec(88903, 13)); + this.add(new SketchBenchmark.Spec(111538, 12)); + this.add(new SketchBenchmark.Spec(136481, 11)); + this.add(new SketchBenchmark.Spec(178605, 6)); + this.add(new SketchBenchmark.Spec(215707, 5)); + this.add(new SketchBenchmark.Spec(273075, 5)); + this.add(new SketchBenchmark.Spec(362878, 5)); + this.add(new SketchBenchmark.Spec(546015, 1)); + this.add(new SketchBenchmark.Spec(1106004, 2)); + this.add(new SketchBenchmark.Spec(1766259, 2)); + }}; + +} diff --git a/src/test/java/com/yahoo/sketches/benchmark/SketchBenchmark.java b/src/test/java/com/yahoo/sketches/benchmark/SketchBenchmark.java new file mode 100644 index 000000000..a3cabb0fb --- /dev/null +++ b/src/test/java/com/yahoo/sketches/benchmark/SketchBenchmark.java @@ -0,0 +1,33 @@ +package com.yahoo.sketches.benchmark; + +import java.util.List; + +/** + */ +public interface SketchBenchmark +{ + public void setup(int numSketches, List specs); + public void runNTimes(int n); + public void reset(); + + public class Spec { + private final int numSketches; + private final long numEntries; + + public Spec(long numEntries, int numSketches) { + + this.numSketches = numSketches; + this.numEntries = numEntries; + } + + public int getNumSketches() + { + return numSketches; + } + + public long getNumEntries() + { + return numEntries; + } + } +} diff --git a/src/test/java/com/yahoo/sketches/benchmark/ThetaBenchmark.java b/src/test/java/com/yahoo/sketches/benchmark/ThetaBenchmark.java new file mode 100644 index 000000000..93583ec81 --- /dev/null +++ b/src/test/java/com/yahoo/sketches/benchmark/ThetaBenchmark.java @@ -0,0 +1,82 @@ +package com.yahoo.sketches.benchmark; + +import com.yahoo.sketches.Family; +import com.yahoo.sketches.theta.CompactSketch; +import com.yahoo.sketches.theta.SetOperation; +import com.yahoo.sketches.theta.Sketch; +import com.yahoo.sketches.theta.Union; +import com.yahoo.sketches.theta.UpdateSketch; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Random; + +/** + */ +public class ThetaBenchmark implements SketchBenchmark +{ + private final int nominalEntries; + private final Random rand; + + private List sketches; + + public ThetaBenchmark(int K) { + this.nominalEntries = 1 << K; + this.rand = new Random(K); + } + + @Override + public void setup(int numSketches, List specs) + { + sketches = new ArrayList<>(numSketches); + + for (Spec spec : specs) { + for (int i = 0; i < spec.getNumSketches(); ++i) { + UpdateSketch sketch = UpdateSketch.builder().build(nominalEntries); + for (int j = 0; j < spec.getNumEntries(); ++j) { + sketch.update(rand.nextLong()); + } + + sketches.add(sketch.rebuild().compact(true, null)); + } + } + Collections.shuffle(sketches, rand); + + int numRetained = 0; + int numEstimating = 0; + for (CompactSketch sketch : sketches) { + numRetained += sketch.getRetainedEntries(true); + if (sketch.isEstimationMode()) { + ++numEstimating; + } + } + System.out.printf( + "%,d entries, %,d/sketch, %,d estimating (%.2f%%)%n", + numRetained, numRetained / sketches.size(), numEstimating, (100 * numEstimating) / (double) sketches.size() + ); + } + + @Override + public void runNTimes(int n) + { + for (int i = 0; i < n; ++i) { + Union combined = (Union) SetOperation.builder().build(nominalEntries, Family.UNION); + for (Object toUnion : sketches) { + combined.update((Sketch) toUnion); + } + } + } + + @Override + public void reset() + { + sketches = null; + } + + @Override + public String toString() + { + return String.format("Theta OnHeap Benchmark(nominalEntries=%s)", nominalEntries); + } +} diff --git a/src/test/java/com/yahoo/sketches/benchmark/ThetaMemoryBenchmark.java b/src/test/java/com/yahoo/sketches/benchmark/ThetaMemoryBenchmark.java new file mode 100644 index 000000000..a98e9e492 --- /dev/null +++ b/src/test/java/com/yahoo/sketches/benchmark/ThetaMemoryBenchmark.java @@ -0,0 +1,88 @@ +package com.yahoo.sketches.benchmark; + +import com.yahoo.sketches.Family; +import com.yahoo.sketches.memory.Memory; +import com.yahoo.sketches.memory.NativeMemory; +import com.yahoo.sketches.theta.SetOperation; +import com.yahoo.sketches.theta.Sketch; +import com.yahoo.sketches.theta.Union; +import com.yahoo.sketches.theta.UpdateSketch; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Random; + +/** + */ +public class ThetaMemoryBenchmark implements SketchBenchmark +{ + private final int nominalEntries; + private final Random rand; + private final byte[] bytes; + + private List memories; + + public ThetaMemoryBenchmark(int K) { + this.nominalEntries = 1 << K; + this.rand = new Random(K); + this.bytes = new byte[Sketch.getMaxUpdateSketchBytes(nominalEntries) + 8]; + } + + @Override + public void setup(int numSketches, List specs) + { + memories = new ArrayList<>(numSketches); + + for (Spec spec : specs) { + for (int i = 0; i < spec.getNumSketches(); ++i) { + UpdateSketch sketch = UpdateSketch.builder().build(nominalEntries); + for (int j = 0; j < spec.getNumEntries(); ++j) { + sketch.update(rand.nextLong()); + } + memories.add(new NativeMemory(sketch.rebuild().compact(true, null).toByteArray())); + } + } + Collections.shuffle(memories, rand); + + int numRetained = 0; + int numEstimating = 0; + for (Memory mem : memories) { + Sketch sketch = Sketch.wrap(mem); + numRetained += sketch.getRetainedEntries(true); + if (sketch.isEstimationMode()) { + ++numEstimating; + } + } + System.out.printf( + "%,d entries, %,d/sketch, %,d estimating (%.2f%%)%n", + numRetained, numRetained / memories.size(), numEstimating, (100 * numEstimating) / (double) memories.size() + ); + } + + @Override + public void runNTimes(int n) + { + for (int i = 0; i < n; ++i) { + Union combined = (Union) SetOperation + .builder() + .setMemory(new NativeMemory(bytes)) + .build(nominalEntries, Family.UNION); + for (Memory toUnion : memories) { + combined.update(toUnion); + } + } + } + + @Override + public void reset() + { + memories = null; + } + + @Override + public String toString() + { + return String.format("Theta Memory Benchmark(nominalEntries=%s)", nominalEntries); + } +}