Skip to content
Permalink
Browse files

Add inode paths to journal

Add inode path to file create, delete and rename journal entries. This
enables path related operations based on only journal entries.

pr-link: #9048
change-id: cid-d5311a8d4a0a4235c45452db7d9bcfd14a6843cc
  • Loading branch information...
bf8086 authored and alluxio-bot committed May 14, 2019
1 parent 33911c0 commit 6264882c1b998c35b512ce4e5350e1789e665639
@@ -1182,6 +1182,7 @@ private void completeFileInternal(RpcContext rpcContext, LockedInodePath inodePa
}
Builder entry = UpdateInodeFileEntry.newBuilder()
.setId(inode.getId())
.setPath(inodePath.getUri().getPath())
.setCompleted(true)
.setLength(length);

@@ -2051,6 +2052,8 @@ private void renameInternal(RpcContext rpcContext, LockedInodePath srcInodePath,
.setOpTimeMs(context.getOperationTimeMs())
.setNewParentId(dstParentInode.getId())
.setNewName(dstName)
.setPath(srcPath.getPath())
.setNewPath(dstPath.getPath())
.build());

// 3. Do UFS operations if necessary.
@@ -2109,6 +2112,8 @@ private void renameInternal(RpcContext rpcContext, LockedInodePath srcInodePath,
.setOpTimeMs(context.getOperationTimeMs())
.setNewName(srcName)
.setNewParentId(srcParentInode.getId())
.setPath(dstPath.getPath())
.setNewPath(srcPath.getPath())
.build());
throw t;
}
@@ -61,6 +61,7 @@

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
@@ -163,6 +164,9 @@ public boolean isWrite() {
/** Only the root inode should have the empty string as its name. */
public static final String ROOT_INODE_NAME = "";

/** Path of the root inode. */
public static final String ROOT_PATH = "/";

/** Number of retries when trying to lock a path, from a given id. */
public static final int PATH_TRAVERSAL_RETRIES = 1000;

@@ -230,7 +234,7 @@ public void initializeRoot(String owner, String group, Mode mode, JournalContext
.mergeFrom(CreateDirectoryPOptions.newBuilder().setMode(mode.toProto()))
.setOwner(owner).setGroup(group));
root.setPersistenceState(PersistenceState.PERSISTED);
mState.applyAndJournal(context, root);
mState.applyAndJournal(context, root, ROOT_PATH);
}
}

@@ -681,6 +685,9 @@ public InodeDirectory getRoot() {
missingDirContext.setMountPoint(false);
missingDirContext.setOwner(context.getOwner());
missingDirContext.setGroup(context.getGroup());
StringBuilder pathBuilder = new StringBuilder().append(
String.join(AlluxioURI.SEPARATOR, Arrays.asList(pathComponents).subList(0, pathIndex))
);
for (int k = pathIndex; k < (pathComponents.length - 1); k++) {
MutableInodeDirectory newDir = MutableInodeDirectory.create(
mDirectoryIdGenerator.getNewDirectoryId(rpcContext.getJournalContext()),
@@ -703,7 +710,10 @@ public InodeDirectory getRoot() {
newDir.setInternalAcl(pair.getFirst());
newDir.setDefaultACL(pair.getSecond());
}
mState.applyAndJournal(rpcContext, newDir);
String newDirPath = k == 0 ? ROOT_PATH
: pathBuilder.append(AlluxioURI.SEPARATOR).append(pathComponents[k]).toString();
mState.applyAndJournal(rpcContext, newDir,
newDirPath);

inodePath.addNextInode(Inode.wrap(newDir));

@@ -786,7 +796,8 @@ public InodeDirectory getRoot() {
}
newInode.setPinned(currentInodeDirectory.isPinned());

mState.applyAndJournal(rpcContext, newInode);
mState.applyAndJournal(rpcContext, newInode,
inodePath.getUri().getPath());
Inode inode = Inode.wrap(newInode);
inodePath.addNextInode(inode);
createdInodes.add(inode);
@@ -860,6 +871,7 @@ public void deleteInode(RpcContext rpcContext, LockedInodePath inodePath, long o
.setId(inode.getId())
.setRecursive(false)
.setOpTimeMs(opTimeMs)
.setPath(inodePath.getUri().getPath())
.build());

if (inode.isFile()) {
@@ -293,11 +293,14 @@ public void applyAndJournal(Supplier<JournalContext> context, UpdateInodeFileEnt
*
* @param context journal context supplier
* @param inode an inode to add and create a journal entry for
* @param path path of the new inode
*/
public void applyAndJournal(Supplier<JournalContext> context, MutableInode<?> inode) {
public void applyAndJournal(Supplier<JournalContext> context, MutableInode<?> inode,
String path) {
try {
applyCreateInode(inode);
context.get().append(inode.toJournalEntry());
context.get().append(inode.toJournalEntry(
Preconditions.checkNotNull(path)));
} catch (Throwable t) {
ProcessUtils.fatalError(LOG, t, "Failed to apply %s", inode);
throw t; // fatalError will usually system.exit
@@ -15,6 +15,7 @@
import alluxio.grpc.TtlAction;
import alluxio.master.ProtobufUtils;
import alluxio.proto.journal.File.UpdateInodeEntry;
import alluxio.proto.journal.Journal;
import alluxio.proto.meta.InodeMeta;
import alluxio.proto.meta.InodeMeta.InodeOrBuilder;
import alluxio.security.authorization.AccessControlList;
@@ -544,6 +545,12 @@ public boolean equals(Object o) {
return mId == that.mId;
}

/**
* @param path path of the inode
* @return the journal entry representing the inode
*/
public abstract Journal.JournalEntry toJournalEntry(String path);

protected MoreObjects.ToStringHelper toStringHelper() {
return MoreObjects.toStringHelper(this)
.add("id", mId)
@@ -248,6 +248,12 @@ public JournalEntry toJournalEntry() {
return JournalEntry.newBuilder().setInodeDirectory(inodeDirectory).build();
}

@Override
public JournalEntry toJournalEntry(String path) {
return JournalEntry.newBuilder().setInodeDirectory(
toJournalEntry().toBuilder().getInodeDirectoryBuilder().setPath(path)).build();
}

@Override
public InodeMeta.Inode toProto() {
return super.toProtoBuilder()
@@ -454,6 +454,12 @@ public JournalEntry toJournalEntry() {
return JournalEntry.newBuilder().setInodeFile(inodeFile).build();
}

@Override
public JournalEntry toJournalEntry(String path) {
return JournalEntry.newBuilder().setInodeFile(
toJournalEntry().toBuilder().getInodeFileBuilder().setPath(path)).build();
}

@Override
public InodeMeta.Inode toProto() {
return super.toProtoBuilder()
@@ -109,37 +109,37 @@ public void after() throws Exception {
@Test
public void writeEvents() throws Exception {
createFile("/file");
assertNotEquals(-1, findFile("file"));
assertNotEquals(INVALID_ID, findFile("file"));

createFile("/for_nested_file/nested_file");
assertNotEquals(-1, findDir("for_nested_file"));
assertNotEquals(-1, findFile("nested_file"));
assertNotEquals(INVALID_ID, findDir("for_nested_file"));
assertNotEquals(INVALID_ID, findFile("nested_file"));

createDir("/dir");
assertNotEquals(-1, findDir("dir"));
assertNotEquals(INVALID_ID, findDir("dir"));

createDir("/for_nested_dir/nested_dir");
assertNotEquals(-1, findDir("for_nested_dir"));
assertNotEquals(-1, findDir("nested_dir"));
assertNotEquals(INVALID_ID, findDir("for_nested_dir"));
assertNotEquals(INVALID_ID, findDir("nested_dir"));

createFile("/rename_src");
mFileSystemMaster.rename(new AlluxioURI("/rename_src"), new AlluxioURI("/rename_dst"),
RenameContext.defaults());
assertNotEquals(-1, findFile("rename_src"));
assertNotEquals(-1, findRename("rename_dst"));
assertNotEquals(INVALID_ID, findFile("rename_src"));
assertNotEquals(INVALID_ID, findRename("rename_dst"));

createFile("/deleted_file");
mFileSystemMaster.delete(new AlluxioURI("/deleted_file"), DeleteContext.defaults());
final long deleteId = findFile("deleted_file");
assertNotEquals(-1, deleteId);
assertNotEquals(-1, findDelete(deleteId));
assertNotEquals(INVALID_ID, deleteId);
assertNotEquals(INVALID_ID, findDelete(deleteId));

createFile("/deleted_dir/file1");
mFileSystemMaster.delete(new AlluxioURI("/deleted_dir"),
DeleteContext.create(DeletePOptions.newBuilder().setRecursive(true)));
final long deleteId1 = findFile("file1");
assertNotEquals(-1, deleteId1);
assertNotEquals(-1, findDelete(deleteId1));
assertNotEquals(INVALID_ID, deleteId1);
assertNotEquals(INVALID_ID, findDelete(deleteId1));
}

@Test
@@ -210,6 +210,70 @@ public void replayEvents() throws Exception {
standbyEntries);
}

@Test
public void writeInodePaths() throws Exception {
String path = "/file";
createFile(path);
final long createId1 = findFile("file", path);
assertNotEquals(INVALID_ID, createId1);
assertNotEquals(INVALID_ID, findCompleteFile(createId1, path));

path = "/for_nested_file/nested_file";
createFile(path);
assertNotEquals(INVALID_ID, findDir("for_nested_file",
new AlluxioURI(path).getParent().getPath()));
final long createId2 = findFile("nested_file", path);
assertNotEquals(INVALID_ID, createId2);
assertNotEquals(INVALID_ID, findCompleteFile(createId2, path));

path = "/dir";
createDir(path);
assertNotEquals(INVALID_ID, findDir("dir", path));

path = "/for_nested_dir/nested_dir";
createDir(path);
assertNotEquals(INVALID_ID, findDir("for_nested_dir",
new AlluxioURI(path).getParent().getPath()));
assertNotEquals(INVALID_ID, findDir("nested_dir", path));

path = "/for_nested_dir/nested_dir/nested_dir_2";
createDir(path);
assertNotEquals(INVALID_ID, findDir("nested_dir_2", path));

path = "/for_nested_dir/nested_dir_3/nested_dir_4";
createDir(path);
assertNotEquals(INVALID_ID, findDir("nested_dir_3",
new AlluxioURI(path).getParent().getPath()));
assertNotEquals(INVALID_ID, findDir("nested_dir_4", path));

path = "/rename_src";
String newPath = "/rename_dst";
createFile(path);
mFileSystemMaster.rename(new AlluxioURI(path), new AlluxioURI(newPath),
RenameContext.defaults());
final long renameId = findFile("rename_src", path);
assertNotEquals(INVALID_ID, renameId);
assertNotEquals(INVALID_ID, findCompleteFile(renameId, path));
assertNotEquals(INVALID_ID, findRename("rename_dst", path, newPath));

path = "/deleted_file";
createFile(path);
mFileSystemMaster.delete(new AlluxioURI(path), DeleteContext.defaults());
final long deleteId = findFile("deleted_file", path);
assertNotEquals(INVALID_ID, deleteId);
assertNotEquals(INVALID_ID, findCompleteFile(deleteId, path));
assertNotEquals(INVALID_ID, findDelete(deleteId, path));

path = "/deleted_dir/file1";
createFile(path);
mFileSystemMaster.delete(new AlluxioURI("/deleted_dir"),
DeleteContext.create(DeletePOptions.newBuilder().setRecursive(true)));
final long deleteId1 = findFile("file1", path);
assertNotEquals(INVALID_ID, deleteId1);
assertNotEquals(INVALID_ID, findCompleteFile(deleteId1, path));
assertNotEquals(INVALID_ID, findDelete(deleteId1, path));
}

private long findFile(String name) throws Exception {
while (!mEntries.isEmpty()) {
JournalEntry entry = mEntries.poll();
@@ -220,6 +284,17 @@ private long findFile(String name) throws Exception {
return INVALID_ID;
}

private long findFile(String name, String path) throws Exception {
while (!mEntries.isEmpty()) {
JournalEntry entry = mEntries.poll();
if (entry.hasInodeFile() && entry.getInodeFile().getName().equals(name)
&& entry.getInodeFile().getPath().equals(path)) {
return entry.getInodeFile().getId();
}
}
return INVALID_ID;
}

private long findDir(String name) throws Exception {
while (!mEntries.isEmpty()) {
JournalEntry entry = mEntries.poll();
@@ -230,6 +305,17 @@ private long findDir(String name) throws Exception {
return INVALID_ID;
}

private long findDir(String name, String path) throws Exception {
while (!mEntries.isEmpty()) {
JournalEntry entry = mEntries.poll();
if (entry.hasInodeDirectory() && entry.getInodeDirectory().getName().equals(name)
&& entry.getInodeDirectory().getPath().equals(path)) {
return entry.getInodeDirectory().getId();
}
}
return INVALID_ID;
}

private long findRename(String newName) throws Exception {
while (!mEntries.isEmpty()) {
JournalEntry entry = mEntries.poll();
@@ -240,6 +326,30 @@ private long findRename(String newName) throws Exception {
return INVALID_ID;
}

private long findRename(String newName, String path, String newPath) throws Exception {
while (!mEntries.isEmpty()) {
JournalEntry entry = mEntries.poll();
if (entry.hasRename() && entry.getRename().getNewName().equals(newName)
&& entry.getRename().getNewPath().equals(newPath)
&& entry.getRename().getPath().equals(path)) {
return entry.getRename().getId();
}
}
return INVALID_ID;
}

private long findCompleteFile(long id, String path) throws Exception {
while (!mEntries.isEmpty()) {
JournalEntry entry = mEntries.poll();
if (entry.hasUpdateInodeFile() && entry.getUpdateInodeFile().getId() == id
&& entry.getUpdateInodeFile().getCompleted()
&& entry.getUpdateInodeFile().getPath().equals(path)) {
return entry.getUpdateInodeFile().getId();
}
}
return INVALID_ID;
}

private long findDelete(long id) throws Exception {
while (!mEntries.isEmpty()) {
JournalEntry entry = mEntries.poll();
@@ -250,6 +360,17 @@ private long findDelete(long id) throws Exception {
return INVALID_ID;
}

private long findDelete(long id, String path) throws Exception {
while (!mEntries.isEmpty()) {
JournalEntry entry = mEntries.poll();
if (entry.hasDeleteFile() && entry.getDeleteFile().getId() == id
&& entry.getDeleteFile().getPath().equals(path)) {
return entry.getDeleteFile().getId();
}
}
return INVALID_ID;
}

private void createFile(String path) throws Exception {
CreateFileContext createFileContext = CreateFileContext
.create(CreateFilePOptions.newBuilder().setRecursive(true).setBlockSizeBytes(1024));

0 comments on commit 6264882

Please sign in to comment.
You can’t perform that action at this time.