Skip to content
Permalink
Browse files

Add option for skipping caching and eviction for scanning inode tree

Scanning all a large portion of the inode tree can thrash metastore when
using the CachingInodeStore. Frequent eviction would negatively impact
the performance of the cache.

This PR introduces an option to disable caching and eviction when
getting inodes from CachingInodeStore.

pr-link: #9062
change-id: cid-9a61f283347e725638c712351880d65fcb51891c
  • Loading branch information...
cheng-chang authored and alluxio-bot committed May 15, 2019
1 parent be06317 commit 784fe198a68bef520c737b799ac31226b000751d
@@ -36,53 +36,53 @@ public DelegatingReadOnlyInodeStore(InodeStore delegate) {
}

@Override
public Optional<Inode> get(long id) {
return mDelegate.get(id);
public Optional<Inode> get(long id, ReadOption option) {
return mDelegate.get(id, option);
}

@Override
public Iterable<Long> getChildIds(Long inodeId) {
return mDelegate.getChildIds(inodeId);
public Iterable<Long> getChildIds(Long inodeId, ReadOption option) {
return mDelegate.getChildIds(inodeId, option);
}

@Override
public Iterable<Long> getChildIds(InodeDirectoryView inode) {
return mDelegate.getChildIds(inode);
public Iterable<Long> getChildIds(InodeDirectoryView inode, ReadOption option) {
return mDelegate.getChildIds(inode, option);
}

@Override
public Iterable<? extends Inode> getChildren(Long inodeId) {
return mDelegate.getChildren(inodeId);
public Iterable<? extends Inode> getChildren(Long inodeId, ReadOption option) {
return mDelegate.getChildren(inodeId, option);
}

@Override
public Iterable<? extends Inode> getChildren(InodeDirectoryView inode) {
return mDelegate.getChildren(inode);
public Iterable<? extends Inode> getChildren(InodeDirectoryView inode, ReadOption option) {
return mDelegate.getChildren(inode, option);
}

@Override
public Optional<Long> getChildId(Long inodeId, String name) {
return mDelegate.getChildId(inodeId, name);
public Optional<Long> getChildId(Long inodeId, String name, ReadOption option) {
return mDelegate.getChildId(inodeId, name, option);
}

@Override
public Optional<Long> getChildId(InodeDirectoryView inode, String name) {
return mDelegate.getChildId(inode, name);
public Optional<Long> getChildId(InodeDirectoryView inode, String name, ReadOption option) {
return mDelegate.getChildId(inode, name, option);
}

@Override
public Optional<Inode> getChild(Long inodeId, String name) {
return mDelegate.getChild(inodeId, name);
public Optional<Inode> getChild(Long inodeId, String name, ReadOption option) {
return mDelegate.getChild(inodeId, name, option);
}

@Override
public Optional<Inode> getChild(InodeDirectoryView inode, String name) {
return mDelegate.getChild(inode, name);
public Optional<Inode> getChild(InodeDirectoryView inode, String name, ReadOption option) {
return mDelegate.getChild(inode, name, option);
}

@Override
public boolean hasChildren(InodeDirectoryView inode) {
return mDelegate.hasChildren(inode);
public boolean hasChildren(InodeDirectoryView inode, ReadOption option) {
return mDelegate.hasChildren(inode, option);
}

@Override
@@ -45,13 +45,22 @@
* inode.
*
* @param id an inode id
* @param option read options
* @return the inode with the given id, if it exists
*/
Optional<MutableInode<?>> getMutable(long id);
Optional<MutableInode<?>> getMutable(long id, ReadOption option);

/**
* @param id an inode id
* @return the result of {@link #getMutable(long, ReadOption)} with default option
*/
default Optional<MutableInode<?>> getMutable(long id) {
return getMutable(id, ReadOption.defaults());
}

@Override
default Optional<Inode> get(long id) {
return getMutable(id).map(Inode::wrap);
default Optional<Inode> get(long id, ReadOption option) {
return getMutable(id, option).map(Inode::wrap);
}

/**
@@ -28,29 +28,55 @@
* Read-only access to the inode store.
*/
public interface ReadOnlyInodeStore extends Closeable {

/**
* @param id an inode id
* @param option the options
* @return the inode with the given id, if it exists
*/
Optional<Inode> get(long id);
Optional<Inode> get(long id, ReadOption option);

/**
* @param id an inode id
* @return the result of {@link #get(long, ReadOption)} with default option
*/
default Optional<Inode> get(long id) {
return get(id, ReadOption.defaults());
}

/**
* Returns an iterable for the ids of the children of the given directory.
*
* @param inodeId an inode id to list child ids for
* @param option the options
* @return the child ids iterable
*/
Iterable<Long> getChildIds(Long inodeId);
Iterable<Long> getChildIds(Long inodeId, ReadOption option);

/**
* @param inodeId an inode id to list child ids for
* @return the result of {@link #getChildIds(Long, ReadOption)} with default option
*/
default Iterable<Long> getChildIds(Long inodeId) {
return getChildIds(inodeId, ReadOption.defaults());
}

/**
* Returns an iterable for the ids of the children of the given directory.
*
* @param inode the inode to list child ids for
* @param option the options
* @return the child ids iterable
*/
default Iterable<Long> getChildIds(InodeDirectoryView inode, ReadOption option) {
return getChildIds(inode.getId(), option);
}

/**
* @param inode the inode to list child ids for
* @return the result of {@link #getChildIds(InodeDirectoryView, ReadOption)} with default option
*/
default Iterable<Long> getChildIds(InodeDirectoryView inode) {
return getChildIds(inode.getId());
return getChildIds(inode, ReadOption.defaults());
}

/**
@@ -61,11 +87,12 @@
* concurrently added inodes will be included.
*
* @param inodeId an inode id
* @param option the options
* @return an iterable over the children of the inode with the given id
*/
default Iterable<? extends Inode> getChildren(Long inodeId) {
default Iterable<? extends Inode> getChildren(Long inodeId, ReadOption option) {
return () -> {
Iterator<Long> it = getChildIds(inodeId).iterator();
Iterator<Long> it = getChildIds(inodeId, option).iterator();
return new Iterator<Inode>() {
private Inode mNext = null;

@@ -90,7 +117,7 @@ void advance() {
while (mNext == null && it.hasNext()) {
Long nextId = it.next();
// Make sure the inode metadata still exists
Optional<Inode> nextInode = get(nextId);
Optional<Inode> nextInode = get(nextId, option);
if (nextInode.isPresent()) {
mNext = nextInode.get();
}
@@ -100,51 +127,119 @@ void advance() {
};
}

/**
* @param inodeId an inode id
* @return the result of {@link #getChildren(Long, ReadOption)} with default option
*/
default Iterable<? extends Inode> getChildren(Long inodeId) {
return getChildren(inodeId, ReadOption.defaults());
}

/**
* @param inode an inode directory
* @param option the options
* @return an iterable over the children of the inode with the given id
*/
default Iterable<? extends Inode> getChildren(InodeDirectoryView inode, ReadOption option) {
return getChildren(inode.getId(), option);
}

/**
* @param inode an inode directory
* @return the result of {@link #getChildren(InodeDirectoryView, ReadOption)} with default option
*/
default Iterable<? extends Inode> getChildren(InodeDirectoryView inode) {
return getChildren(inode.getId());
return getChildren(inode.getId(), ReadOption.defaults());
}

/**
* @param inodeId an inode id
* @param name an inode name
* @param option the options
* @return the id of the child of the inode with the given name
*/
Optional<Long> getChildId(Long inodeId, String name);
Optional<Long> getChildId(Long inodeId, String name, ReadOption option);

/**
* @param inodeId an inode id
* @param name an inode name
* @return the result of {@link #getChildId(Long, String, ReadOption)} with default option
*/
default Optional<Long> getChildId(Long inodeId, String name) {
return getChildId(inodeId, name, ReadOption.defaults());
}

/**
* @param inode an inode directory
* @param name an inode name
* @param option the options
* @return the id of the child of the inode with the given name
*/
default Optional<Long> getChildId(InodeDirectoryView inode, String name, ReadOption option) {
return getChildId(inode.getId(), name, option);
}

/**
* @param inode an inode directory
* @param name an inode name
* @return the result of {@link #getChildId(InodeDirectoryView, String, ReadOption)} with default
* option
*/
default Optional<Long> getChildId(InodeDirectoryView inode, String name) {
return getChildId(inode.getId(), name);
return getChildId(inode.getId(), name, ReadOption.defaults());
}

/**
* @param inodeId an inode id
* @param name an inode name
* @param option the options
* @return the child of the inode with the given name
*/
Optional<Inode> getChild(Long inodeId, String name);
Optional<Inode> getChild(Long inodeId, String name, ReadOption option);

/**
* @param inodeId an inode id
* @param name an inode name
* @return the result of {@link #getChild(Long, String, ReadOption)} with default option
*/
default Optional<Inode> getChild(Long inodeId, String name) {
return getChild(inodeId, name, ReadOption.defaults());
}

/**
* @param inode an inode directory
* @param name an inode name
* @param option the options
* @return the child of the inode with the given name
*/
default Optional<Inode> getChild(InodeDirectoryView inode, String name, ReadOption option) {
return getChild(inode.getId(), name, option);
}

/**
* @param inode an inode directory
* @param name an inode name
* @return the result of {@link #getChild(InodeDirectoryView, String, ReadOption)} with default
* option
*/
default Optional<Inode> getChild(InodeDirectoryView inode, String name) {
return getChild(inode.getId(), name);
return getChild(inode.getId(), name, ReadOption.defaults());
}

/**
* @param inode an inode directory
* @param option the options
* @return whether the inode has any children
*/
boolean hasChildren(InodeDirectoryView inode);
boolean hasChildren(InodeDirectoryView inode, ReadOption option);

/**
* @param inode an inode directory
* @return the result of {@link #hasChildren(InodeDirectoryView, ReadOption)} with default option
*/
default boolean hasChildren(InodeDirectoryView inode) {
return hasChildren(inode, ReadOption.defaults());
}

/**
* @return all edges in the inode store
@@ -0,0 +1,71 @@
/*
* The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
* (the "License"). You may not use this work except in compliance with the License, which is
* available at www.apache.org/licenses/LICENSE-2.0
*
* This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied, as more fully set forth in the License.
*
* See the NOTICE file distributed with this work for information regarding copyright ownership.
*/

package alluxio.master.metastore;

/**
* Options for reading from the inode store.
*/
public class ReadOption {
private static final ReadOption DEFAULT = new ReadOption(false);

private final boolean mSkipCache;

private ReadOption(boolean skipCache) {
mSkipCache = skipCache;
}

/**
* @return whether to skip caching when reading from the inode store
*/
public boolean shouldSkipCache() {
return mSkipCache;
}

/**
* @return a new builder
*/
public static Builder newBuilder() {
return new Builder();
}

/**
* @return the singleton instance of the default option
*/
public static ReadOption defaults() {
return DEFAULT;
}

/**
* Builder for {@link ReadOption}.
*/
public static class Builder {
private boolean mSkipCache = false;

/**
* Sets whether to skip caching.
*
* @param skip skip or not
* @return the builder
*/
public Builder setSkipCache(boolean skip) {
mSkipCache = skip;
return this;
}

/**
* @return the built option
*/
public ReadOption build() {
return new ReadOption(mSkipCache);
}
}
}

0 comments on commit 784fe19

Please sign in to comment.
You can’t perform that action at this time.