From 7039402d212a9539410d16e7530774ea6e9744eb Mon Sep 17 00:00:00 2001 From: Bowen Li Date: Tue, 2 Jan 2018 11:21:28 -0800 Subject: [PATCH 1/3] update local branch --- update.sh | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 update.sh diff --git a/update.sh b/update.sh new file mode 100644 index 0000000000000..709a2780bb05a --- /dev/null +++ b/update.sh @@ -0,0 +1,2 @@ +git fetch upstream +git rebase upstream/master From 55981e19de89aeeaeb06f37786f9346071651e6f Mon Sep 17 00:00:00 2001 From: Bowen Li Date: Wed, 3 Jan 2018 17:35:11 -0800 Subject: [PATCH 2/3] remove sh --- update.sh | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 update.sh diff --git a/update.sh b/update.sh deleted file mode 100644 index 709a2780bb05a..0000000000000 --- a/update.sh +++ /dev/null @@ -1,2 +0,0 @@ -git fetch upstream -git rebase upstream/master From 39eb2687902cede79159920264bfdf1dab23be4a Mon Sep 17 00:00:00 2001 From: Bowen Li Date: Sat, 20 Jan 2018 16:23:06 -0800 Subject: [PATCH 3/3] [FLINK-8469] unify RocksDB params in RocksDBPerformanceTest --- .../benchmark/RocksDBPerformanceTest.java | 90 +++++++++---------- 1 file changed, 40 insertions(+), 50 deletions(-) diff --git a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/benchmark/RocksDBPerformanceTest.java b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/benchmark/RocksDBPerformanceTest.java index 533196e6e7b02..e05e7ae2f16e6 100644 --- a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/benchmark/RocksDBPerformanceTest.java +++ b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/benchmark/RocksDBPerformanceTest.java @@ -24,6 +24,8 @@ import org.apache.flink.testutils.junit.RetryRule; import org.apache.flink.util.TestLogger; +import org.junit.After; +import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -36,6 +38,7 @@ import sun.misc.Unsafe; import java.io.File; +import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Arrays; @@ -51,37 +54,43 @@ public class RocksDBPerformanceTest extends TestLogger { @Rule public final RetryRule retry = new RetryRule(); - @Test(timeout = 2000) - @RetryOnFailure(times = 3) - public void testRocksDbMergePerformance() throws Exception { - final File rocksDir = tmp.newFolder(); + private File rocksDir; + private Options options; + private WriteOptions writeOptions; - // ensure the RocksDB library is loaded to a distinct location each retry - NativeLibraryLoader.getInstance().loadLibrary(rocksDir.getAbsolutePath()); + private final String key = "key"; + private final String value = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ7890654321"; - final String key = "key"; - final String value = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ7890654321"; + private final byte[] keyBytes = key.getBytes(StandardCharsets.UTF_8); + private final byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8); - final byte[] keyBytes = key.getBytes(StandardCharsets.UTF_8); - final byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8); + @Before + public void init() throws IOException { + rocksDir = tmp.newFolder(); - final int num = 50000; + // ensure the RocksDB library is loaded to a distinct location each retry + NativeLibraryLoader.getInstance().loadLibrary(rocksDir.getAbsolutePath()); - try ( - final Options options = new Options() - .setCompactionStyle(CompactionStyle.LEVEL) - .setLevelCompactionDynamicLevelBytes(true) - .setIncreaseParallelism(4) - .setUseFsync(false) - .setMaxOpenFiles(-1) - .setCreateIfMissing(true) - .setMergeOperatorName(RocksDBKeyedStateBackend.MERGE_OPERATOR_NAME); + options = new Options() + .setCompactionStyle(CompactionStyle.LEVEL) + .setLevelCompactionDynamicLevelBytes(true) + .setIncreaseParallelism(4) + .setUseFsync(false) + .setMaxOpenFiles(-1) + .setCreateIfMissing(true) + .setMergeOperatorName(RocksDBKeyedStateBackend.MERGE_OPERATOR_NAME); + + writeOptions = new WriteOptions() + .setSync(false) + .setDisableWAL(true); + } - final WriteOptions writeOptions = new WriteOptions() - .setSync(false) - .setDisableWAL(true); + @Test(timeout = 2000) + @RetryOnFailure(times = 3) + public void testRocksDbMergePerformance() throws Exception { + final int num = 50000; - final RocksDB rocksDB = RocksDB.open(options, rocksDir.getAbsolutePath())) { + try (RocksDB rocksDB = RocksDB.open(options, rocksDir.getAbsolutePath())) { // ----- insert ----- log.info("begin insert"); @@ -131,34 +140,9 @@ public void testRocksDbMergePerformance() throws Exception { @Test(timeout = 2000) @RetryOnFailure(times = 3) public void testRocksDbRangeGetPerformance() throws Exception { - final File rocksDir = tmp.newFolder(); - - // ensure the RocksDB library is loaded to a distinct location each retry - NativeLibraryLoader.getInstance().loadLibrary(rocksDir.getAbsolutePath()); - - final String key = "key"; - final String value = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ7890654321"; - - final byte[] keyBytes = key.getBytes(StandardCharsets.UTF_8); - final byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8); - final int num = 50000; - try ( - final Options options = new Options() - .setCompactionStyle(CompactionStyle.LEVEL) - .setLevelCompactionDynamicLevelBytes(true) - .setIncreaseParallelism(4) - .setUseFsync(false) - .setMaxOpenFiles(-1) - .setCreateIfMissing(true) - .setMergeOperatorName(RocksDBKeyedStateBackend.MERGE_OPERATOR_NAME); - - final WriteOptions writeOptions = new WriteOptions() - .setSync(false) - .setDisableWAL(true); - - final RocksDB rocksDB = RocksDB.open(options, rocksDir.getAbsolutePath())) { + try (RocksDB rocksDB = RocksDB.open(options, rocksDir.getAbsolutePath())) { final byte[] keyTemplate = Arrays.copyOf(keyBytes, keyBytes.length + 4); @@ -211,4 +195,10 @@ private static boolean samePrefix(byte[] prefix, byte[] key) { return true; } + + @After + public void close() { + options.close(); + writeOptions.close(); + } }