From a6340a62366766a444ad20bb10cba2893f9be098 Mon Sep 17 00:00:00 2001 From: runzhiwang Date: Mon, 23 Mar 2020 20:33:15 +0800 Subject: [PATCH 1/4] HDDS-3244. Improve write efficiency by opening RocksDB only once --- .../container/common/impl/HddsDispatcher.java | 25 +++++++++++ .../common/utils/ContainerCache.java | 29 +++++++++++++ .../keyvalue/helpers/BlockUtils.java | 42 +++++++++++++++++++ .../helpers/KeyValueContainerUtil.java | 10 ----- 4 files changed, 96 insertions(+), 10 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java index cef1c8f8fb5..1242d793c6c 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java @@ -52,6 +52,8 @@ import org.apache.hadoop.ozone.container.common.transport.server.ratis .DispatcherContext; import org.apache.hadoop.ozone.container.common.volume.VolumeSet; +import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils; +import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .ContainerCommandRequestProto; @@ -163,6 +165,20 @@ public ContainerCommandResponseProto dispatch( } } + private void createRocksDBAndPutCache(Container container) + throws IOException { + if (container == null) { + return; + } + + KeyValueContainerData kvContainerData = + (KeyValueContainerData)container.getContainerData(); + if (!BlockUtils.isDBExist(kvContainerData, conf)) { + BlockUtils.createDBAndPutCache(kvContainerData.getContainerDBType(), + kvContainerData.getDbFile().getAbsolutePath(), conf); + } + } + @SuppressWarnings("methodlength") private ContainerCommandResponseProto dispatchRequest( ContainerCommandRequestProto msg, DispatcherContext dispatcherContext) { @@ -213,6 +229,15 @@ private ContainerCommandResponseProto dispatchRequest( container2BCSIDMap = dispatcherContext.getContainer2BCSIDMap(); } if (isWriteCommitStage) { + try { + createRocksDBAndPutCache(container); + } catch (IOException ioe) { + StorageContainerException sce = new StorageContainerException( + "Create RocksDB failed. " + ioe.getMessage(), ioe, + Result.UNABLE_TO_READ_METADATA_DB); + return ContainerUtils.logAndReturnError(LOG, sce, msg); + } + // check if the container Id exist in the loaded snapshot file. if // it does not , it infers that , this is a restart of dn where // the we are reapplying the transaction which was not captured in the diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java index 4ddb4e48792..2671e00f18c 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java @@ -160,4 +160,33 @@ public void removeDB(String containerDBPath) { lock.unlock(); } } + + /** + * Check whether a DB handler exists in cache. + * + * @param containerDBPath - DB path of the container. + */ + public boolean isDBExist(String containerDBPath) { + lock.lock(); + try { + return this.get(containerDBPath) != null; + } finally { + lock.unlock(); + } + } + + /** + * Add a DB handler into cache. + * + * @param containerDBPath - DB path of the container. + * @param db - DB handler + */ + public void addDB(String containerDBPath, ReferenceCountedDB db) { + lock.lock(); + try { + this.putIfAbsent(containerDBPath, db); + } finally { + lock.unlock(); + } + } } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/BlockUtils.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/BlockUtils.java index 35e0b0c15a7..e715710e053 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/BlockUtils.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/BlockUtils.java @@ -22,11 +22,14 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException; +import org.apache.hadoop.hdds.utils.MetadataStore; +import org.apache.hadoop.hdds.utils.MetadataStoreBuilder; import org.apache.hadoop.ozone.container.common.helpers.BlockData; import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData; import org.apache.hadoop.ozone.container.common.utils.ContainerCache; import org.apache.hadoop.ozone.container.common.utils.ReferenceCountedDB; +import java.io.File; import java.io.IOException; import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos @@ -95,6 +98,45 @@ public static void shutdownCache(ContainerCache cache) { cache.shutdownCache(); } + /** + * Check whether a DB handler exists in cache. + * + * @param containerData containerData. + * @param conf configuration. + */ + public static boolean isDBExist(KeyValueContainerData containerData, + Configuration conf) { + Preconditions.checkNotNull(containerData); + ContainerCache cache = ContainerCache.getInstance(conf); + Preconditions.checkNotNull(cache); + Preconditions.checkNotNull(containerData.getDbFile()); + return cache.isDBExist(containerData.getDbFile().getAbsolutePath()); + } + + /** + * Create a DB handler and put it into cache. + * + * @param containerDBType - DB type of the container. + * @param containerDBPath - DB path of the container. + * @param conf configuration. + */ + public static void createDBAndPutCache(String containerDBType, + String containerDBPath, Configuration conf) throws IOException { + ContainerCache cache = ContainerCache.getInstance(conf); + Preconditions.checkNotNull(cache); + + MetadataStore metadataStore = + MetadataStoreBuilder.newBuilder() + .setDbFile(new File(containerDBPath)) + .setCreateIfMissing(true) + .setConf(conf) + .setDBType(containerDBType) + .build(); + ReferenceCountedDB db = + new ReferenceCountedDB(metadataStore, containerDBPath); + cache.addDB(containerDBPath, db); + } + /** * Parses the {@link BlockData} from a bytes array. * diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java index 501680ae84e..34f779e4f0a 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java @@ -32,8 +32,6 @@ import org.apache.hadoop.ozone.container.common.helpers.BlockData; import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData; import org.apache.hadoop.hdds.utils.MetadataKeyFilters; -import org.apache.hadoop.hdds.utils.MetadataStore; -import org.apache.hadoop.hdds.utils.MetadataStoreBuilder; import com.google.common.base.Preconditions; import org.apache.commons.io.FileUtils; @@ -73,14 +71,6 @@ public static void createContainerMetaData(File containerMetaDataPath, File throw new IOException("Unable to create directory for metadata storage." + " Path: " + containerMetaDataPath); } - MetadataStore store = MetadataStoreBuilder.newBuilder().setConf(conf) - .setCreateIfMissing(true).setDbFile(dbFile).build(); - - // we close since the SCM pre-creates containers. - // we will open and put Db handle into a cache when keys are being created - // in a container. - - store.close(); if (!chunksPath.mkdirs()) { LOG.error("Unable to create chunks directory Container {}", From a145399c4d979244698d9dbef966671bc7e45cba Mon Sep 17 00:00:00 2001 From: runzhiwang Date: Fri, 27 Mar 2020 17:41:42 +0800 Subject: [PATCH 2/4] HDDS-3244. Open Rocksdb in createContainer --- .../container/common/impl/HddsDispatcher.java | 25 ------------- .../common/utils/ContainerCache.java | 14 -------- .../keyvalue/helpers/BlockUtils.java | 36 +++---------------- .../helpers/KeyValueContainerUtil.java | 9 +++++ 4 files changed, 13 insertions(+), 71 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java index 1242d793c6c..cef1c8f8fb5 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java @@ -52,8 +52,6 @@ import org.apache.hadoop.ozone.container.common.transport.server.ratis .DispatcherContext; import org.apache.hadoop.ozone.container.common.volume.VolumeSet; -import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils; -import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .ContainerCommandRequestProto; @@ -165,20 +163,6 @@ public ContainerCommandResponseProto dispatch( } } - private void createRocksDBAndPutCache(Container container) - throws IOException { - if (container == null) { - return; - } - - KeyValueContainerData kvContainerData = - (KeyValueContainerData)container.getContainerData(); - if (!BlockUtils.isDBExist(kvContainerData, conf)) { - BlockUtils.createDBAndPutCache(kvContainerData.getContainerDBType(), - kvContainerData.getDbFile().getAbsolutePath(), conf); - } - } - @SuppressWarnings("methodlength") private ContainerCommandResponseProto dispatchRequest( ContainerCommandRequestProto msg, DispatcherContext dispatcherContext) { @@ -229,15 +213,6 @@ private ContainerCommandResponseProto dispatchRequest( container2BCSIDMap = dispatcherContext.getContainer2BCSIDMap(); } if (isWriteCommitStage) { - try { - createRocksDBAndPutCache(container); - } catch (IOException ioe) { - StorageContainerException sce = new StorageContainerException( - "Create RocksDB failed. " + ioe.getMessage(), ioe, - Result.UNABLE_TO_READ_METADATA_DB); - return ContainerUtils.logAndReturnError(LOG, sce, msg); - } - // check if the container Id exist in the loaded snapshot file. if // it does not , it infers that , this is a restart of dn where // the we are reapplying the transaction which was not captured in the diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java index 2671e00f18c..04eb30fe2a8 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java @@ -161,20 +161,6 @@ public void removeDB(String containerDBPath) { } } - /** - * Check whether a DB handler exists in cache. - * - * @param containerDBPath - DB path of the container. - */ - public boolean isDBExist(String containerDBPath) { - lock.lock(); - try { - return this.get(containerDBPath) != null; - } finally { - lock.unlock(); - } - } - /** * Add a DB handler into cache. * diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/BlockUtils.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/BlockUtils.java index e715710e053..7ceb622eb43 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/BlockUtils.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/BlockUtils.java @@ -22,14 +22,11 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException; -import org.apache.hadoop.hdds.utils.MetadataStore; -import org.apache.hadoop.hdds.utils.MetadataStoreBuilder; import org.apache.hadoop.ozone.container.common.helpers.BlockData; import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData; import org.apache.hadoop.ozone.container.common.utils.ContainerCache; import org.apache.hadoop.ozone.container.common.utils.ReferenceCountedDB; -import java.io.File; import java.io.IOException; import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos @@ -99,41 +96,16 @@ public static void shutdownCache(ContainerCache cache) { } /** - * Check whether a DB handler exists in cache. + * Add a DB handler into cache. * - * @param containerData containerData. - * @param conf configuration. - */ - public static boolean isDBExist(KeyValueContainerData containerData, - Configuration conf) { - Preconditions.checkNotNull(containerData); - ContainerCache cache = ContainerCache.getInstance(conf); - Preconditions.checkNotNull(cache); - Preconditions.checkNotNull(containerData.getDbFile()); - return cache.isDBExist(containerData.getDbFile().getAbsolutePath()); - } - - /** - * Create a DB handler and put it into cache. - * - * @param containerDBType - DB type of the container. + * @param db - DB handler. * @param containerDBPath - DB path of the container. * @param conf configuration. */ - public static void createDBAndPutCache(String containerDBType, - String containerDBPath, Configuration conf) throws IOException { + public static void addDB(ReferenceCountedDB db, String containerDBPath, + Configuration conf) { ContainerCache cache = ContainerCache.getInstance(conf); Preconditions.checkNotNull(cache); - - MetadataStore metadataStore = - MetadataStoreBuilder.newBuilder() - .setDbFile(new File(containerDBPath)) - .setCreateIfMissing(true) - .setConf(conf) - .setDBType(containerDBType) - .build(); - ReferenceCountedDB db = - new ReferenceCountedDB(metadataStore, containerDBPath); cache.addDB(containerDBPath, db); } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java index 34f779e4f0a..8580cc9d70e 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java @@ -32,6 +32,8 @@ import org.apache.hadoop.ozone.container.common.helpers.BlockData; import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData; import org.apache.hadoop.hdds.utils.MetadataKeyFilters; +import org.apache.hadoop.hdds.utils.MetadataStore; +import org.apache.hadoop.hdds.utils.MetadataStoreBuilder; import com.google.common.base.Preconditions; import org.apache.commons.io.FileUtils; @@ -72,6 +74,13 @@ public static void createContainerMetaData(File containerMetaDataPath, File " Path: " + containerMetaDataPath); } + MetadataStore store = MetadataStoreBuilder.newBuilder().setConf(conf) + .setCreateIfMissing(true).setDbFile(dbFile).build(); + ReferenceCountedDB db = + new ReferenceCountedDB(store, dbFile.getAbsolutePath()); + //add db handler into cache + BlockUtils.addDB(db, dbFile.getAbsolutePath(), conf); + if (!chunksPath.mkdirs()) { LOG.error("Unable to create chunks directory Container {}", chunksPath); From 01853b20d9fcdcd4660052ee7aace4ee19e6aee1 Mon Sep 17 00:00:00 2001 From: runzhiwang Date: Thu, 2 Apr 2020 09:47:11 +0800 Subject: [PATCH 3/4] retrigger build with empty commit From 58096de61eaff93221180d6cc74dea55bdb9bfb7 Mon Sep 17 00:00:00 2001 From: runzhiwang Date: Thu, 2 Apr 2020 17:40:59 +0800 Subject: [PATCH 4/4] fix code review --- .../keyvalue/helpers/KeyValueContainerUtil.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java index 8580cc9d70e..d4ea45950dd 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java @@ -74,13 +74,6 @@ public static void createContainerMetaData(File containerMetaDataPath, File " Path: " + containerMetaDataPath); } - MetadataStore store = MetadataStoreBuilder.newBuilder().setConf(conf) - .setCreateIfMissing(true).setDbFile(dbFile).build(); - ReferenceCountedDB db = - new ReferenceCountedDB(store, dbFile.getAbsolutePath()); - //add db handler into cache - BlockUtils.addDB(db, dbFile.getAbsolutePath(), conf); - if (!chunksPath.mkdirs()) { LOG.error("Unable to create chunks directory Container {}", chunksPath); @@ -90,6 +83,13 @@ public static void createContainerMetaData(File containerMetaDataPath, File throw new IOException("Unable to create directory for data storage." + " Path: " + chunksPath); } + + MetadataStore store = MetadataStoreBuilder.newBuilder().setConf(conf) + .setCreateIfMissing(true).setDbFile(dbFile).build(); + ReferenceCountedDB db = + new ReferenceCountedDB(store, dbFile.getAbsolutePath()); + //add db handler into cache + BlockUtils.addDB(db, dbFile.getAbsolutePath(), conf); } /**